From 4c65bc2a1036d5558b7614718fa546e42f8ef1d5 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Mon, 16 May 2022 21:15:46 -0700 Subject: [PATCH 01/53] feat: SDK support for model monitoring (for models deployed to endpoints; batch prediction use case will be implemented separately)" --- .../cloud/aiplatform/compat/types/__init__.py | 8 + google/cloud/aiplatform/jobs.py | 339 +++++++++++++++++- .../aiplatform/model_monitoring/__init__.py | 31 ++ .../aiplatform/model_monitoring/alert.py | 67 ++++ .../aiplatform/model_monitoring/objective.py | 216 +++++++++++ .../aiplatform/model_monitoring/sampling.py | 49 +++ .../aiplatform/model_monitoring/schedule.py | 49 +++ 7 files changed, 758 insertions(+), 1 deletion(-) create mode 100644 google/cloud/aiplatform/model_monitoring/__init__.py create mode 100644 google/cloud/aiplatform/model_monitoring/alert.py create mode 100644 google/cloud/aiplatform/model_monitoring/objective.py create mode 100644 google/cloud/aiplatform/model_monitoring/sampling.py create mode 100644 google/cloud/aiplatform/model_monitoring/schedule.py diff --git a/google/cloud/aiplatform/compat/types/__init__.py b/google/cloud/aiplatform/compat/types/__init__.py index 14ff93f011..d3ec99a6c4 100644 --- a/google/cloud/aiplatform/compat/types/__init__.py +++ b/google/cloud/aiplatform/compat/types/__init__.py @@ -59,7 +59,9 @@ model as model_v1beta1, model_evaluation as model_evaluation_v1beta1, model_evaluation_slice as model_evaluation_slice_v1beta1, + model_deployment_monitoring_job as model_deployment_monitoring_job_v1beta1, model_service as model_service_v1beta1, + model_monitoring as model_monitoring_v1beta1, operation as operation_v1beta1, pipeline_job as pipeline_job_v1beta1, pipeline_service as pipeline_service_v1beta1, @@ -120,7 +122,9 @@ model as model_v1, model_evaluation as model_evaluation_v1, model_evaluation_slice as model_evaluation_slice_v1, + model_deployment_monitoring_job as model_deployment_monitoring_job_v1, model_service as model_service_v1, + model_monitoring as model_monitoring_v1, operation as operation_v1, pipeline_job as pipeline_job_v1, pipeline_service as pipeline_service_v1, @@ -183,7 +187,9 @@ model_v1, model_evaluation_v1, model_evaluation_slice_v1, + model_deployment_monitoring_job_v1, model_service_v1, + model_monitoring_v1, operation_v1, pipeline_job_v1, pipeline_service_v1, @@ -243,7 +249,9 @@ model_v1beta1, model_evaluation_v1beta1, model_evaluation_slice_v1beta1, + model_deployment_monitoring_job_v1beta1, model_service_v1beta1, + model_monitoring_v1beta1, operation_v1beta1, pipeline_job_v1beta1, pipeline_service_v1beta1, diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 00d6f11780..489e28bb88 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -36,12 +36,16 @@ completion_stats as gca_completion_stats, custom_job as gca_custom_job_compat, explanation as gca_explanation_compat, + encryption_spec as gca_encryption_spec_compat, io as gca_io_compat, job_state as gca_job_state, hyperparameter_tuning_job as gca_hyperparameter_tuning_job_compat, machine_resources as gca_machine_resources_compat, manual_batch_tuning_parameters as gca_manual_batch_tuning_parameters_compat, + study as gca_study_compat, + model_deployment_monitoring_job as gca_model_deployment_monitoring_job_compat, + model_monitoring as gca_model_monitoring_compat, ) from google.cloud.aiplatform.constants import base as constants from google.cloud.aiplatform import initializer @@ -50,6 +54,7 @@ from google.cloud.aiplatform.utils import console_utils from google.cloud.aiplatform.utils import source_utils from google.cloud.aiplatform.utils import worker_spec_utils +from google.cloud.aiplatform.compat.services import endpoint_service_client _LOGGER = base.Logger(__name__) @@ -378,6 +383,7 @@ def create( sync: bool = True, create_request_timeout: Optional[float] = None, batch_size: Optional[int] = None, + ) -> "BatchPredictionJob": """Create a batch prediction job. @@ -664,7 +670,7 @@ def create( gapic_batch_prediction_job.manual_batch_tuning_parameters = ( manual_batch_tuning_parameters ) - + # User Labels gapic_batch_prediction_job.labels = labels @@ -1912,3 +1918,334 @@ def run( def trials(self) -> List[gca_study_compat.Trial]: self._assert_gca_resource_is_available() return list(self._gca_resource.trials) + + +class ModelDeploymentMonitoringJob(_Job): + """Vertex AI Model Deployment Monitoring Job. + + This class should be used in conjunction with the Endpoint class + in order to configure model monitoring for deployed models. + """ + + _resource_noun = "modelDeploymentMonitoringJobs" + _getter_method = "get_model_deployment_monitoring_job" + _list_method = "list_model_deployment_monitoring_jobs" + _cancel_method = None + _delete_method = "delete_model_deployment_monitoring_job" + _job_type = "model-deployment-monitoring" + _parse_resource_name_method = "parse_model_deployment_monitoring_job_path" + _format_resource_name_method = "model_deployment_monitoring_job_path" + + def __init__( + self, + model_deployment_monitoring_job_name: str, + project: Optional[str] = None, + location: Optional[str] = None, + credentials: Optional[auth_credentials.Credentials] = None, + ): + """Initializer for ModelDeploymentMonitoringJob""" + super().__init__( + job_name=model_deployment_monitoring_job_name, + project=project, + location=location, + credentials=credentials, + ) + + + def _parse_configs(objective_configs: Union[ + model_monitoring.objective.EndpointObjectiveConfig, + Dict[str, model_monitoring.objective.EndpointObjectiveConfig]): + + all_configs = {} + all_models = [] + client_options = dict(api_endpoint=API_ENDPOINT) + client = endpoint_service_client(client_options=client_options) + parent = f"projects/{self.project}/locations/{self.location}" + response = client.get_endpoint(name=f"{parent}/endpoints/{endpoint}") + for model in response.deployed_models: + all_models.append(model) + + ## when same objective config is applied to ALL models + if isinstance(objective_configs, model_monitoring.objective.EndpointObjectiveConfig) and deployed_model_ids is None: + for model in all_models: + all_configs[model] = objective_configs + + ## when same objective config is applied to SOME models + elif isinstance(objective_configs, model_monitoring.objective.EndpointObjectiveConfig) and isinstance(deployed_model_ids, List): + for model in deployed_model_ids: + assert(model in all_models) + all_configs[model] = objective_configs + + ## when different objective configs are applied to EACH model + elif isinstance(objective_configs, Dict) and deployed_model_ids is None: + assert(all(model in all_models for model in objective_configs.keys())) + all_configs = objective_configs + + mdm_objective_config_seq = [] + for key in all_configs.keys(): + mdm_objective_config_seq.append( + gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringObjectiveConfig( + deployed_model_id = key, + objective_config = all_configs[key] + )) + return mdm_objective_config_seq + + @classmethod + def create( + cls, + display_name: str, + endpoint: Union[str, "models.Endpoint"], + objective_configs: Union[ + model_monitoring.objective.EndpointObjectiveConfig, + Dict[str, model_monitoring.objective.EndpointObjectiveConfig]], + logging_sampling_strategy: model_monitoring.sampling._SamplingStrategy, + monitor_interval: int, + deployed_model_ids: Optional[List[str]] = ["*"], + schedule_config: model_monitoring.schedule._ScheduleConfig, + alert_config: Optional[model_monitoring.alert._AlertConfig] = None, + predict_instance_schema_uri: Optional[str] = None, + sample_predict_instance: Optional[str] = None, + analysis_instance_schema_uri: Optional[str] = None, + bigquery_tables_log_ttl: Optional[int] = None, + stats_anomalies_base_directory: Optional[str] = None, + enable_monitoring_pipeline_logs: Optional[bool] = None, + labels: Optional[Dict[str, str]] = None, + encryption_spec_key_name: Optional[str] = None, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + project: Optional[str] = None, + location: Optional[str] = None, + credentials: Optional[auth_credentials.Credentials] = None, + sync: bool = True, + ) -> "ModelDeploymentMonitoringJob": + """Creates and launches a model monitoring job + + Args: + display_name (str): + The user-defined name of the + ModelDeploymentMonitoringJob. The name can be up + to 128 characters long and can be consist of any + UTF-8 characters. + Display name of a ModelDeploymentMonitoringJob. + + endpoint (Union[str, "models.Endpoint"]): + Endpoint resource name. Format: + ``projects/{project}/locations/{location}/endpoints/{endpoint}`` + + objective_configs (Union[ + model_monitoring.objective.EndpointObjectiveConfig, + Dict[str, model_monitoring.objective.EndpointObjectiveConfig]): + A single config if it applies to all models, or a dictionary of + model_id: model_monitoring.objective.EndpointObjectiveConfig if + different model IDs have different configs + + logging_sampling_strategy (model_monitoring.sampling._SamplingStrategy): + Sample Strategy for logging. + + monitor_interval (int): + The model monitoring job scheduling + interval. It will be rounded up to next full + hour. This defines how often the monitoring jobs + are triggered. + + deployed_model_ids ([List[str]] = ["*"]): + Optional. Use this argument to specify which deployed models to + apply the objective config to. If left unspecified, the same config + will be applied to all deployed models. + + schedule_config (model_monitoring.schedule._ScheduleConfig): + Configures model monitoring job scheduling interval in hours. + This defines how often the monitoring jobs are triggered. + + alert_config (model_monitoring.alert._AlertConfig): + Optional. Configures how alerts are sent to the user. Right now + only email alert is supported. + + predict_instance_schema_uri (str): + Optional. YAML schema file uri describing the format of + a single instance, which are given to format + the Endpoint's prediction (and explanation). If + not set, the schema will be generated from + collected predict requests. + + sample_predict_instance (str): + Sample Predict instance, same format as PredictionRequest.instances, + this can be set as a replacement of predict_instance_schema_uri + If not set, the schema will be generated from collected predict requests. + + analysis_instance_schema_uri (str): + YAML schema file uri describing the format of a single + instance that you want Tensorflow Data Validation (TFDV) to + analyze. If this field is empty, all the feature data types are + inferred from predict_instance_schema_uri, meaning that TFDV + will use the data in the exact format as prediction request/response. + If there are any data type differences between predict instance + and TFDV instance, this field can be used to override the schema. + For models trained with Vertex AI, this field must be set as all the + fields in predict instance formatted as string. + + bigquery_tables_log_ttl (int): + The TTL(time to live) of BigQuery tables in user projects + which stores logs. A day is the basic unit of + the TTL and we take the ceil of TTL/86400(a + day). e.g. { second: 3600} indicates ttl = 1 + day. + + stats_anomalies_base_directory (str): + Optional. Stats anomalies base folder path. + + enable_monitoring_pipeline_logs (bool): + Optional. If true, the scheduled monitoring pipeline logs are sent to + Google Cloud Logging, including pipeline status and + anomalies detected. Please note the logs incur cost, which + are subject to `Cloud Logging + pricing `__. + + labels (Dict[str, str]): + Optional. The labels with user-defined metadata to + organize the ModelDeploymentMonitoringJob. + Label keys and values can be no longer than 64 + characters (Unicode codepoints), can only + contain lowercase letters, numeric characters, + underscores and dashes. International characters + are allowed. See https://goo.gl/xmQnxf for more information + and examples of labels. + + encryption_spec_key_name (str): + Optional. Customer-managed encryption key spec for a + ModelDeploymentMonitoringJob. If set, this + ModelDeploymentMonitoringJob and all + sub-resources of this + ModelDeploymentMonitoringJob will be secured by + this key. + + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + + project: Optional[str] = None, + location: Optional[str] = None, + credentials: Optional[auth_credentials.Credentials] = None, + sync: bool = True + + Returns: + An instance of ModelDeploymentMonitoringJob + + """ + if not display_name: + display_name = cls._generate_display_name() + + utils.validate_display_name(display_name) + + if labels: + utils.validate_labels(labels) + + if stats_anomalies_base_directory: + stats_anomalies_base_directory = gca_io_compat.GcsDestination( + output_uri_prefix = stats_anomalies_base_directory) + + if encryption_spec_key_name: + encryption_spec_key_name = gca_encryption_spec_compat.EncryptionSpec( + kms_key_name = encryption_spec_key_name) + + mdm_objective_config_seq = self._parse_configs(objective_configs) + + self._gca_resource = gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringJob( + name=self.model_deployment_monitoring_job_name, + display_name=display_name, + endpoint=endpoint, + model_deployment_monitoring_objective_configs=mdm_objective_config_seq, + logging_sampling_strategy=logging_sampling_strategy, + model_deployment_monitoring_schedule_config=schedule_config, + model_monitoring_alert_config=alerting_config, + predict_instance_schema_uri=predict_instance_schema_uri, + analysis_instance_schema_uri=analysis_instance_schema_uri, + sample_predict_instance = sample_predict_instance, + stats_anomalies_base_directory = stats_anomalies_base_directory, + enable_monitoring_pipeline_logs = enable_monitoring_pipeline_logs, + labels = labels, + encryption_spec = encryption_spec_key_name + ) + + api_client = self.api_client + mdm_job = api_client.create_model_deployment_monitoring_job( + parent = parent, + model_deployment_monitoring_job = self._gca_resource + ) + return mdm_job + + + def update( + self, + display_name: Optional[str] = None, + objective_configs: Optional[Union[ + model_monitoring.objective.EndpointObjectiveConfig, + Dict[str, model_monitoring.objective.EndpointObjectiveConfig]] = None, + logging_sampling_strategy: \ + Optional[model_monitoring.sampling._SamplingStrategy] = None, + monitor_interval: Optional[int] = None, + deployed_model_ids: Optional[List[str]] = None, + schedule_config: Optional[model_monitoring.schedule._ScheduleConfig] = None, + alert_config: Optional[model_monitoring.alert._AlertConfig] = None, + predict_instance_schema_uri: Optional[str] = None, + sample_predict_instance: Optional[str] = None, + analysis_instance_schema_uri: Optional[str] = None, + bigquery_tables_log_ttl: Optional[int] = None, + stats_anomalies_base_directory: Optional[str] = None, + enable_monitoring_pipeline_logs: Optional[bool] = None, + labels: Optional[Dict[str, str]] = None, + encryption_spec_key_name: Optional[str] = None, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + project: Optional[str] = None, + location: Optional[str] = None, + credentials: Optional[auth_credentials.Credentials] = None, + sync: bool = True, + ) -> "ModelDeploymentMonitoringJob": + """""" + current_job = self.api_client.get_model_deployment_monitoring_job( + name = slef.model_deployment_monitoring_job_name) + update_mask: List[str] = [] + if display_name: + update_mask.append("display_name") + current_job.display_name = display_name + if schedule_config: + update_mask.append("model_deployment_monitoring_schedule_config") + current_job.model_deployment_monitoring_schedule_config = schedule_config + if alert_config: + update_mask.append("model_monitoring_alert_config") + current_job.model_monitoring_alert_config = alert_config + if logging_sampling_strategy: + update_mask.append("logging_sampling_strategy") + current_job.logging_sampling_strategy = logging_sampling_strategy + if labels: + update_mask.append("labels") + current_job.lables = labels + if bigquery_tables_log_ttl: + update_mask.append("log_ttl") + current_job.log_ttl = bigquery_tables_log_ttl + if enable_monitoring_pipeline_logs: + update_mask.append("enable_monitoring_pipeline_logs") + current_job.enable_monitoring_pipeline_logs = enable_monitoring_pipeline_logs + if objective_configs: + update_mask.append("model_deployment_monitoring_objective_configs") + current_job.model_deployment_monitoring_objective_configs = self._parse_configs(objective_configs) + self.api_client.update_model_deployment_monitoring_job( + model_deployment_monitoring_job = current_job, + update_mask = update_mask + ) + + + def pause(self) -> "ModelDeploymentMonitoringJob": + """""" + self.api_client.pause_model_deployment_monitoring_job( + self.model_deployment_monitoring_job_name) + + def resume(self) -> "ModelDeploymentMonitoringJob": + """""" + self.api_client.resume_model_deployment_monitoring_job( + self.model_deployment_monitoring_job_name) + + def delete(self) -> "ModelDeploymentMonitoringJob": + """""" + self.api_client.delete_model_deployment_monitoring_job( + self.model_deployment_monitoring_job_name) diff --git a/google/cloud/aiplatform/model_monitoring/__init__.py b/google/cloud/aiplatform/model_monitoring/__init__.py new file mode 100644 index 0000000000..cf86822ada --- /dev/null +++ b/google/cloud/aiplatform/model_monitoring/__init__.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from google.cloud.aiplatform.model_monitoring.alert import EmailAlertConfig +from google.cloud.aiplatform.model_monitoring.objective import EndpointSkewDetectionConfig, EndpointDriftDetectionConfig, EndpointExplanationConfig, EndpointObjectiveConfig +from google.cloud.aiplatform.model_monitoring.sampling import RandomSampleConfig +from google.cloud.aiplatform.model_monitoring.schedule import ScheduleConfig + +__all__ = ( + "EmailAlertConfig", + "EndpointSkewDetectionConfig", + "EndpointDriftDetectionConfig", + "EndpointExplanationConfig", + "EndpointObjectiveConfig", + "RandomSampleConfig", + "ScheduleConfig" +) diff --git a/google/cloud/aiplatform/model_monitoring/alert.py b/google/cloud/aiplatform/model_monitoring/alert.py new file mode 100644 index 0000000000..01514545f1 --- /dev/null +++ b/google/cloud/aiplatform/model_monitoring/alert.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import abc +from google.aiplatform.compat.types import model_monitoring as gca_model_monitoring + +class _AlertConfig(abc.ABC): + """An abstract class for setting model monitoring alert config""" + def __init__( + self, + enable_logging: Optional[bool] = None + ): + self.enable_logging = enable_logging + + @abstractmethod + def as_proto(self) -> gca_model_monitoring.ModelMonitoringAlertConfig: + pass + + +class EmailAlertConfig(_AlertConfig): + def __init__( + self, + user_emails: List[str], + enable_logging: Optional[bool] = None + ): + """Initializer for EmailAlertConfig + + Args: + user_emails (List[str]): + The email addresses to send the alert to. + enable_logging (bool): + Optional. Streams detected anomalies to Cloud Logging. The anomalies will be + put into json payload encoded from proto + [google.cloud.aiplatform.logging.ModelMonitoringAnomaliesLogEntry][]. + This can be further sync'd to Pub/Sub or any other services + supported by Cloud Logging. + + Returns: + An instance of EmailAlertConfig + """ + super().__init__( + enable_logging = self.enable_logging + ) + self.user_emails = user_emails + + def as_proto(self) -> gca_model_monitoring.ModelMonitoringAlertConfig: + user_email_alert_config = gca_model_monitoring.ModelMonitoringAlertConfig.EmailAlertConfig( + user_emails = self.user_emails + ) + return gca_model_monitoring.ModelMonitoringAlertConfig( + email_alert_config = user_email_alert_config, + enable_logging = self.enable_logging + ) diff --git a/google/cloud/aiplatform/model_monitoring/objective.py b/google/cloud/aiplatform/model_monitoring/objective.py new file mode 100644 index 0000000000..a1626b7d47 --- /dev/null +++ b/google/cloud/aiplatform/model_monitoring/objective.py @@ -0,0 +1,216 @@ +# -*- coding: utf-8 -*- + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import abc +from google.aiplatform.compat.types import model_monitoring as gca_model_monitoring +from google.cloud.aiplatform_v1.types import ThresholdConfig as gca_threshold_config + +class _SkewDetectionConfig(abc.ABC): + def __init__( + self, + data_source: str, + skew_thresholds: Dict[str, float], + attribute_skew_thresholds: Optional[Dict[str, float]] = None, + data_format: Optional[str] = None, + target_field: Optional[str] = None, + ): + """""" + self.data_source = data_source + self.skew_thresholds = skew_thresholds + self.attribute_skew_thresholds = attribute_skew_thresholds + self.data_format = data_format + self.target_field = target_field + + def as_proto(self) -> \ +gca_model_monitoring.ModelMonitoringObjectiveConfig.TrainingPredictionSkewDetectionConfig: + skew_thresholds_mapping = {} + attribution_score_skew_thresholds_mapping = {} + for key in self.skew_thresholds.keys(): + skew_threshold = gca_threshold_config.ThresholdConfig(value = self.skew_thresholds[key]) + skew_thresholds_mapping[key] = skew_threshold + for key in self.attribute_skew_thresholds.keys(): + attribution_score_skew_threshold = gca_threshold_config.ThresholdConfig(value = self.attribute_skew_thresholds[key]) + attribution_score_skew_thresholds_mapping[key] = attribution_score_skew_threshold + return gca_model_monitoring.ModelMonitoringObjectiveConfig.TrainingPredictionSkewDetectionConfig( + skew_thresholds = skew_thresholds_mapping, + attribution_score_skew_thresholds = attribution_score_skew_thresholds_mapping + ) + + + +class _DriftDetectionConfig(abc.ABC): + def __init__( + self, + drift_thresholds: Dict[str, float], + attribute_drift_thresholds: Optional[Dict[str, float]] = None, + ): + self.drift_thresholds = drift_thresholds + self.attribute_drift_thresholds = attribute_drift_thresholds + + def as_proto(self) -> \ +gca_model_monitoring.ModelMonitoringObjectiveConfig.PredictionDriftDetectionConfig: + drift_thresholds_mapping = {} + attribution_score_drift_thresholds_mapping = {} + for key in self.drift_thresholds.keys(): + drift_threshold = gca_threshold_config.ThresholdConfig(value = self.drift_thresholds[key]) + drift_thresholds_mapping[key] = drift_threshold + for key in self.attribute_drift_thresholds.keys(): + attribution_score_drift_threshold = gca_threshold_config.ThresholdConfig(value = self.attribute_drift_thresholds[key]) + attribution_score_drift_thresholds_mapping[key] = attribution_score_drift_threshold + return gca_model_monitoring.ModelMonitoringObjectiveConfig.PredictionDriftDetectionConfig( + drift_thresholds = drift_thresholds_mapping, + attribution_score_drift_thresholds = attribution_score_drift_thresholds_mapping + ) + + +class _ExplanationConfig(abc.ABC): + def __init__(self): + self.enable_feature_attributes = False + + def as_proto(self) -> \ + gca_model_monitoring.ModelMonitoringObjectiveConfig.ExplanationConfig: + return gca_model_monitoring.ModelMonitoringObjectiveConfig.ExplanationConfig(enable_feature_attributes = self.enable_feature_attributes) + +class _ObjectiveConfig(abc.ABC): + def __init__( + self, + skew_detection_config: \ + Optional["model_monitoring._SkewDetectionConfig"] = None, + drift_detection_config: \ + Optional["model_monitoring._DriftDetectionConfig"] = None, + explanation_config: \ + Optional["model_monitoring._ExplanationConfig"] = None, + ): + self.skew_detection_config = skew_detection_config + self.drift_detection_config = drift_detection_config + self.explanation_config = explanation_config + + def as_proto(self) -> gca_model_monitoring.ModelMonitoringObjectiveConfig: + training_dataset = None + if skew_detection_config is not None: + training_dataset = skew_detection_config.data_source + return gca_model_monitoring.ModelMonitoringObjectiveConfig( + training_dataset = training_dataset + training_prediction_skew_detection_config = self.skew_detection_config + prediction_drift_detection_config = self.drift_detection_config + explanation = self.explanation_config + ) + + +class EndpointSkewDetectionConfig(_SkewDetectionConfig): + """A class that configures skew detection for models deployed to an endpoint. + + Training-serving skew occurs when input data in production has a different + distribution than the data used during model training. Model performance + can deteriorate when production data deviates from training data. + """ + def __init__( + self, + data_source: str, + skew_thresholds: Optional[Dict[str, float]] = None, + attribute_skew_thresholds: Optional[Dict[str, float]] = None, + data_format: Optional[str] = None, + target_field: Optional[str] = None, + ): + """Initializer for EndpointSkewDetectionConfig + + Args: + data_source (str): + Path to training dataset. + + skew_thresholds (Dict[str, float]): + Optional. Key is the feature name and value is the + threshold. If a feature needs to be monitored + for skew, a value threshold must be configured + for that feature. The threshold here is against + feature distribution distance between the + training and prediction feature. + + attribute_skew_thresholds (Dict[str, float]): + Optional. Key is the feature name and value is the + threshold. Feature attributions indicate how much + each feature in your model contributed to the + predictions for each given instance. + + data_format (str): + Optional. Data format of the dataset, only applicable + if the input is from Google Cloud Storage. + The possible formats are: + + "tf-record" + The source file is a TFRecord file. + + "csv" + The source file is a CSV file. + + target_field (str): + The target field name the model is to + predict. This field will be excluded when doing + Predict and (or) Explain for the training data. + + Returns: + An instance of EndpointSkewDetectionConfig + """ + super().__init__(data_source, skew_threshold, attribute_skew_thresholds, data_format, target_field) + +class EndpointDriftDetectionConfig(_DriftDetectionConfig): + """A class that configures prediction drift detection for models deployed to an endpoint. + + Prediction drift occurs when feature data distribution changes noticeably + over time, and should be set when the original training data is unavailable. + If original training data is available, EndpointSkewDetectionConfig should + be set instead. + """ + def __init__( + self, + drift_thresholds: Optional[Dict[str, float]] = None, + attribute_drift_thresholds: Optional[Dict[str, float]] = None, + ): + """Initializer for EndpointDriftDetectionConfig + + Args: + drift_thresholds (Dict[str, float]): + + attribute_drift_thresholds (Dict[str, float]): + + Returns: + An instance of EndpointDriftDetectionConfig + """ + super().__init__(drift_threshold, attribute_drift_thresholds) + +class EndpointExplanationConfig(_ExplanationConfig): + """A class that enables Vertex Explainable AI. + + Only applicable if the model has explanation_spec populated. + """ + def __init__(self): + super().__init__() + self.enable_feature_attributes = True + +class EndpointObjectiveConfig(_ObjectiveConfig): + """A class that captures skew detection, drift detection, and explaination configs.""" + def __init__( + self, + skew_detection_config: Optional[ + "model_monitoring.EndpointSkewDetectionConfig"] = None, + drift_detection_config: Optional[ + "model_monitoring.EndpointDriftDetectionConfig"] = None, + explanation_config: \ + Optional["model_monitoring.EndpointExplanationConfig"] = None, + ): + """""" + super().__init__(skew_detection_config, drift_detection_config, explanation_config) \ No newline at end of file diff --git a/google/cloud/aiplatform/model_monitoring/sampling.py b/google/cloud/aiplatform/model_monitoring/sampling.py new file mode 100644 index 0000000000..f17eb524b2 --- /dev/null +++ b/google/cloud/aiplatform/model_monitoring/sampling.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import abc +from google.aiplatform.compat.types import model_monitoring as gca_model_monitoring + +class _SamplingStrategy(abc.ABC): + """An abstract class for setting sampling strategy for model monitoring""" + @abstractmethod + def as_proto(self) -> gca_model_monitoring.SamplingStrategy: + pass + +class RandomSampleConfig(_SamplingStrategy): + def __init__( + self, + sample_rate: Optional[float] = None + ): + """Initializer for RandomSampleConfig + + Args: + sample_rate (float): + Optional. Sets the sampling rate for model monitoring logs. + If not set, all logs are processed. + Returns: + An instance of RandomSampleConfig + """ + super.()__init() + self.sample_rate = sample_rate + + def as_proto(self) -> gca_model_monitoring.SamplingStrategy: + return gca_model_monitoring.SamplingStrategy( + random_sample_config = gca_model_monitoring.SamplingStrategy.RandomSampleConfig( + sample_rate = self.sample_rate + ) + ) diff --git a/google/cloud/aiplatform/model_monitoring/schedule.py b/google/cloud/aiplatform/model_monitoring/schedule.py new file mode 100644 index 0000000000..728f8039c9 --- /dev/null +++ b/google/cloud/aiplatform/model_monitoring/schedule.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import abc +from google.protobuf import duration_pb2 # type: ignore +from google.aiplatform.compat.types import model_deployment_monitoring_job as gca_model_deployment_monitoring_job + +class _ScheduleConfig(abc.ABC): + """""" + @abstractmethod + def as_proto(self) -> gca_model_deployment_monitoring_job.ModelDeploymentMonitoringScheduleConfig: + pass + +class ScheduleConfig(_ScheduleConfig): + """""" + def __init__( + self, + monitor_interval: int + ): + """Initializer for ScheduleConfig + + Args: + monitor_interval (int): + Sets the model monitoring job scheduling interval in hours. + This defines how often the monitoring jobs are triggered. + Returns: + An instance of ScheduleConfig + """ + super.()__init__() + self.monitor_interval = monitor_interval + + def as_proto(self) -> gca_model_deployment_monitoring_job.ModelDeploymentMonitoringScheduleConfig: + return gca_model_deployment_monitoring_job.ModelDeploymentMonitoringScheduleConfig( + monitor_interval = duration_pb2.Duration(seconds = self.monitor_interval) + ) From 69b9a074657f4f9511b3ed06d845b1ce3d583a69 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Tue, 24 May 2022 15:14:11 -0700 Subject: [PATCH 02/53] fixing syntax errors --- google/cloud/aiplatform/compat/__init__.py | 4 ++ google/cloud/aiplatform/jobs.py | 55 ++++++++++--------- .../aiplatform/model_monitoring/alert.py | 9 +-- .../aiplatform/model_monitoring/objective.py | 20 +++---- .../aiplatform/model_monitoring/sampling.py | 28 +++++----- .../aiplatform/model_monitoring/schedule.py | 27 ++++----- 6 files changed, 69 insertions(+), 74 deletions(-) diff --git a/google/cloud/aiplatform/compat/__init__.py b/google/cloud/aiplatform/compat/__init__.py index 6aea51d133..c4a71e1beb 100644 --- a/google/cloud/aiplatform/compat/__init__.py +++ b/google/cloud/aiplatform/compat/__init__.py @@ -92,6 +92,8 @@ types.model = types.model_v1beta1 types.model_evaluation = types.model_evaluation_v1beta1 types.model_evaluation_slice = types.model_evaluation_slice_v1beta1 + types.model_deployment_monitoring_job = types.model_deployment_monitoring_job_v1beta1, + types.model_monitoring = types.model_monitoring_v1beta1, types.model_service = types.model_service_v1beta1 types.operation = types.operation_v1beta1 types.pipeline_job = types.pipeline_job_v1beta1 @@ -174,6 +176,8 @@ types.model = types.model_v1 types.model_evaluation = types.model_evaluation_v1 types.model_evaluation_slice = types.model_evaluation_slice_v1 + types.model_deployment_monitoring_job = types.model_deployment_monitoring_job_v1, + types.model_monitoring = types.model_monitoring_v1, types.model_service = types.model_service_v1 types.operation = types.operation_v1 types.pipeline_job = types.pipeline_job_v1 diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 489e28bb88..0e95ec64fd 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -15,7 +15,7 @@ # limitations under the License. # -from typing import Iterable, Optional, Union, Sequence, Dict, List +from typing import Iterable, Optional, Union, Sequence, Dict, List, Tuple import abc import copy @@ -50,6 +50,7 @@ from google.cloud.aiplatform.constants import base as constants from google.cloud.aiplatform import initializer from google.cloud.aiplatform import hyperparameter_tuning +from google.cloud.aiplatform import model_monitoring from google.cloud.aiplatform import utils from google.cloud.aiplatform.utils import console_utils from google.cloud.aiplatform.utils import source_utils @@ -1953,8 +1954,8 @@ def __init__( def _parse_configs(objective_configs: Union[ - model_monitoring.objective.EndpointObjectiveConfig, - Dict[str, model_monitoring.objective.EndpointObjectiveConfig]): + model_monitoring.EndpointObjectiveConfig, + Dict[str, model_monitoring.EndpointObjectiveConfig]]): all_configs = {} all_models = [] @@ -1966,12 +1967,12 @@ def _parse_configs(objective_configs: Union[ all_models.append(model) ## when same objective config is applied to ALL models - if isinstance(objective_configs, model_monitoring.objective.EndpointObjectiveConfig) and deployed_model_ids is None: + if isinstance(objective_configs, model_monitoring.EndpointObjectiveConfig) and deployed_model_ids is None: for model in all_models: all_configs[model] = objective_configs ## when same objective config is applied to SOME models - elif isinstance(objective_configs, model_monitoring.objective.EndpointObjectiveConfig) and isinstance(deployed_model_ids, List): + elif isinstance(objective_configs, model_monitoring.EndpointObjectiveConfig) and isinstance(deployed_model_ids, List): for model in deployed_model_ids: assert(model in all_models) all_configs[model] = objective_configs @@ -1996,13 +1997,16 @@ def create( display_name: str, endpoint: Union[str, "models.Endpoint"], objective_configs: Union[ - model_monitoring.objective.EndpointObjectiveConfig, - Dict[str, model_monitoring.objective.EndpointObjectiveConfig]], - logging_sampling_strategy: model_monitoring.sampling._SamplingStrategy, + model_monitoring.EndpointObjectiveConfig, + Dict[str, model_monitoring.EndpointObjectiveConfig]], + logging_sampling_strategy: model_monitoring.RandomSampleConfig, monitor_interval: int, + schedule_config: model_monitoring.ScheduleConfig, + metadata: Sequence[Tuple[str, str]], + timeout: float = None, + sync: bool = True, deployed_model_ids: Optional[List[str]] = ["*"], - schedule_config: model_monitoring.schedule._ScheduleConfig, - alert_config: Optional[model_monitoring.alert._AlertConfig] = None, + alert_config: Optional[model_monitoring.EmailAlertConfig] = None, predict_instance_schema_uri: Optional[str] = None, sample_predict_instance: Optional[str] = None, analysis_instance_schema_uri: Optional[str] = None, @@ -2011,12 +2015,9 @@ def create( enable_monitoring_pipeline_logs: Optional[bool] = None, labels: Optional[Dict[str, str]] = None, encryption_spec_key_name: Optional[str] = None, - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), project: Optional[str] = None, location: Optional[str] = None, credentials: Optional[auth_credentials.Credentials] = None, - sync: bool = True, ) -> "ModelDeploymentMonitoringJob": """Creates and launches a model monitoring job @@ -2039,7 +2040,7 @@ def create( model_id: model_monitoring.objective.EndpointObjectiveConfig if different model IDs have different configs - logging_sampling_strategy (model_monitoring.sampling._SamplingStrategy): + logging_sampling_strategy (model_monitoring.sampling.RandomSampleConfig): Sample Strategy for logging. monitor_interval (int): @@ -2053,11 +2054,11 @@ def create( apply the objective config to. If left unspecified, the same config will be applied to all deployed models. - schedule_config (model_monitoring.schedule._ScheduleConfig): + schedule_config (model_monitoring.schedule.ScheduleConfig): Configures model monitoring job scheduling interval in hours. This defines how often the monitoring jobs are triggered. - alert_config (model_monitoring.alert._AlertConfig): + alert_config (model_monitoring.alert.EmailAlertConfig): Optional. Configures how alerts are sent to the user. Right now only email alert is supported. @@ -2147,7 +2148,7 @@ def create( encryption_spec_key_name = gca_encryption_spec_compat.EncryptionSpec( kms_key_name = encryption_spec_key_name) - mdm_objective_config_seq = self._parse_configs(objective_configs) + mdm_objective_config_seq = cls._parse_configs(objective_configs) self._gca_resource = gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringJob( name=self.model_deployment_monitoring_job_name, @@ -2166,26 +2167,28 @@ def create( encryption_spec = encryption_spec_key_name ) - api_client = self.api_client + api_client = cls.api_client mdm_job = api_client.create_model_deployment_monitoring_job( parent = parent, - model_deployment_monitoring_job = self._gca_resource + model_deployment_monitoring_job = cls._gca_resource ) return mdm_job def update( self, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), display_name: Optional[str] = None, objective_configs: Optional[Union[ - model_monitoring.objective.EndpointObjectiveConfig, - Dict[str, model_monitoring.objective.EndpointObjectiveConfig]] = None, + model_monitoring.EndpointObjectiveConfig, + Dict[str, model_monitoring.EndpointObjectiveConfig]]] = None, logging_sampling_strategy: \ - Optional[model_monitoring.sampling._SamplingStrategy] = None, + Optional[model_monitoring.RandomSampleConfig] = None, monitor_interval: Optional[int] = None, deployed_model_ids: Optional[List[str]] = None, - schedule_config: Optional[model_monitoring.schedule._ScheduleConfig] = None, - alert_config: Optional[model_monitoring.alert._AlertConfig] = None, + schedule_config: Optional[model_monitoring.ScheduleConfig] = None, + alert_config: Optional[model_monitoring.EmailAlertConfig] = None, predict_instance_schema_uri: Optional[str] = None, sample_predict_instance: Optional[str] = None, analysis_instance_schema_uri: Optional[str] = None, @@ -2194,8 +2197,6 @@ def update( enable_monitoring_pipeline_logs: Optional[bool] = None, labels: Optional[Dict[str, str]] = None, encryption_spec_key_name: Optional[str] = None, - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), project: Optional[str] = None, location: Optional[str] = None, credentials: Optional[auth_credentials.Credentials] = None, @@ -2246,6 +2247,6 @@ def resume(self) -> "ModelDeploymentMonitoringJob": self.model_deployment_monitoring_job_name) def delete(self) -> "ModelDeploymentMonitoringJob": - """""" + """""" self.api_client.delete_model_deployment_monitoring_job( self.model_deployment_monitoring_job_name) diff --git a/google/cloud/aiplatform/model_monitoring/alert.py b/google/cloud/aiplatform/model_monitoring/alert.py index 01514545f1..5a39d88167 100644 --- a/google/cloud/aiplatform/model_monitoring/alert.py +++ b/google/cloud/aiplatform/model_monitoring/alert.py @@ -16,7 +16,8 @@ # import abc -from google.aiplatform.compat.types import model_monitoring as gca_model_monitoring +from typing import Optional, List +from google.cloud.aiplatform.compat.types import model_monitoring as gca_model_monitoring class _AlertConfig(abc.ABC): """An abstract class for setting model monitoring alert config""" @@ -25,10 +26,6 @@ def __init__( enable_logging: Optional[bool] = None ): self.enable_logging = enable_logging - - @abstractmethod - def as_proto(self) -> gca_model_monitoring.ModelMonitoringAlertConfig: - pass class EmailAlertConfig(_AlertConfig): @@ -57,7 +54,7 @@ def __init__( ) self.user_emails = user_emails - def as_proto(self) -> gca_model_monitoring.ModelMonitoringAlertConfig: + def as_proto(self): user_email_alert_config = gca_model_monitoring.ModelMonitoringAlertConfig.EmailAlertConfig( user_emails = self.user_emails ) diff --git a/google/cloud/aiplatform/model_monitoring/objective.py b/google/cloud/aiplatform/model_monitoring/objective.py index a1626b7d47..d2e28454de 100644 --- a/google/cloud/aiplatform/model_monitoring/objective.py +++ b/google/cloud/aiplatform/model_monitoring/objective.py @@ -16,7 +16,8 @@ # import abc -from google.aiplatform.compat.types import model_monitoring as gca_model_monitoring +from typing import Optional, Dict +from google.cloud.aiplatform.compat.types import model_monitoring as gca_model_monitoring from google.cloud.aiplatform_v1.types import ThresholdConfig as gca_threshold_config class _SkewDetectionConfig(abc.ABC): @@ -35,8 +36,7 @@ def __init__( self.data_format = data_format self.target_field = target_field - def as_proto(self) -> \ -gca_model_monitoring.ModelMonitoringObjectiveConfig.TrainingPredictionSkewDetectionConfig: + def as_proto(self): skew_thresholds_mapping = {} attribution_score_skew_thresholds_mapping = {} for key in self.skew_thresholds.keys(): @@ -61,8 +61,7 @@ def __init__( self.drift_thresholds = drift_thresholds self.attribute_drift_thresholds = attribute_drift_thresholds - def as_proto(self) -> \ -gca_model_monitoring.ModelMonitoringObjectiveConfig.PredictionDriftDetectionConfig: + def as_proto(self): drift_thresholds_mapping = {} attribution_score_drift_thresholds_mapping = {} for key in self.drift_thresholds.keys(): @@ -81,8 +80,7 @@ class _ExplanationConfig(abc.ABC): def __init__(self): self.enable_feature_attributes = False - def as_proto(self) -> \ - gca_model_monitoring.ModelMonitoringObjectiveConfig.ExplanationConfig: + def as_proto(self): return gca_model_monitoring.ModelMonitoringObjectiveConfig.ExplanationConfig(enable_feature_attributes = self.enable_feature_attributes) class _ObjectiveConfig(abc.ABC): @@ -99,14 +97,14 @@ def __init__( self.drift_detection_config = drift_detection_config self.explanation_config = explanation_config - def as_proto(self) -> gca_model_monitoring.ModelMonitoringObjectiveConfig: + def as_proto(self): training_dataset = None if skew_detection_config is not None: training_dataset = skew_detection_config.data_source return gca_model_monitoring.ModelMonitoringObjectiveConfig( - training_dataset = training_dataset - training_prediction_skew_detection_config = self.skew_detection_config - prediction_drift_detection_config = self.drift_detection_config + training_dataset = training_dataset, + training_prediction_skew_detection_config = self.skew_detection_config, + prediction_drift_detection_config = self.drift_detection_config, explanation = self.explanation_config ) diff --git a/google/cloud/aiplatform/model_monitoring/sampling.py b/google/cloud/aiplatform/model_monitoring/sampling.py index f17eb524b2..4630963114 100644 --- a/google/cloud/aiplatform/model_monitoring/sampling.py +++ b/google/cloud/aiplatform/model_monitoring/sampling.py @@ -16,32 +16,30 @@ # import abc -from google.aiplatform.compat.types import model_monitoring as gca_model_monitoring +from typing import Optional +from google.cloud.aiplatform.compat.types import model_monitoring as gca_model_monitoring class _SamplingStrategy(abc.ABC): """An abstract class for setting sampling strategy for model monitoring""" - @abstractmethod - def as_proto(self) -> gca_model_monitoring.SamplingStrategy: - pass class RandomSampleConfig(_SamplingStrategy): def __init__( self, sample_rate: Optional[float] = None ): - """Initializer for RandomSampleConfig + """Initializer for RandomSampleConfig - Args: - sample_rate (float): - Optional. Sets the sampling rate for model monitoring logs. - If not set, all logs are processed. - Returns: - An instance of RandomSampleConfig - """ - super.()__init() - self.sample_rate = sample_rate + Args: + sample_rate (float): + Optional. Sets the sampling rate for model monitoring logs. + If not set, all logs are processed. + Returns: + An instance of RandomSampleConfig + """ + super().__init() + self.sample_rate = sample_rate - def as_proto(self) -> gca_model_monitoring.SamplingStrategy: + def as_proto(self): return gca_model_monitoring.SamplingStrategy( random_sample_config = gca_model_monitoring.SamplingStrategy.RandomSampleConfig( sample_rate = self.sample_rate diff --git a/google/cloud/aiplatform/model_monitoring/schedule.py b/google/cloud/aiplatform/model_monitoring/schedule.py index 728f8039c9..b761d5bf15 100644 --- a/google/cloud/aiplatform/model_monitoring/schedule.py +++ b/google/cloud/aiplatform/model_monitoring/schedule.py @@ -17,13 +17,10 @@ import abc from google.protobuf import duration_pb2 # type: ignore -from google.aiplatform.compat.types import model_deployment_monitoring_job as gca_model_deployment_monitoring_job +from google.cloud.aiplatform.compat.types import model_deployment_monitoring_job as gca_model_deployment_monitoring_job class _ScheduleConfig(abc.ABC): """""" - @abstractmethod - def as_proto(self) -> gca_model_deployment_monitoring_job.ModelDeploymentMonitoringScheduleConfig: - pass class ScheduleConfig(_ScheduleConfig): """""" @@ -31,19 +28,19 @@ def __init__( self, monitor_interval: int ): - """Initializer for ScheduleConfig + """Initializer for ScheduleConfig - Args: - monitor_interval (int): - Sets the model monitoring job scheduling interval in hours. - This defines how often the monitoring jobs are triggered. - Returns: - An instance of ScheduleConfig - """ - super.()__init__() - self.monitor_interval = monitor_interval + Args: + monitor_interval (int): + Sets the model monitoring job scheduling interval in hours. + This defines how often the monitoring jobs are triggered. + Returns: + An instance of ScheduleConfig + """ + super().__init__() + self.monitor_interval = monitor_interval - def as_proto(self) -> gca_model_deployment_monitoring_job.ModelDeploymentMonitoringScheduleConfig: + def as_proto(self): return gca_model_deployment_monitoring_job.ModelDeploymentMonitoringScheduleConfig( monitor_interval = duration_pb2.Duration(seconds = self.monitor_interval) ) From 10a9201ce4757c18a8202df4e0c7281b457ce7a8 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 24 May 2022 22:16:53 +0000 Subject: [PATCH 03/53] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- google/cloud/aiplatform/compat/__init__.py | 10 +- google/cloud/aiplatform/jobs.py | 124 ++++++++++-------- .../aiplatform/model_monitoring/__init__.py | 9 +- .../aiplatform/model_monitoring/alert.py | 35 +++-- .../aiplatform/model_monitoring/objective.py | 102 +++++++++----- .../aiplatform/model_monitoring/sampling.py | 27 ++-- .../aiplatform/model_monitoring/schedule.py | 30 +++-- 7 files changed, 198 insertions(+), 139 deletions(-) diff --git a/google/cloud/aiplatform/compat/__init__.py b/google/cloud/aiplatform/compat/__init__.py index c4a71e1beb..57e2abdf6b 100644 --- a/google/cloud/aiplatform/compat/__init__.py +++ b/google/cloud/aiplatform/compat/__init__.py @@ -92,8 +92,10 @@ types.model = types.model_v1beta1 types.model_evaluation = types.model_evaluation_v1beta1 types.model_evaluation_slice = types.model_evaluation_slice_v1beta1 - types.model_deployment_monitoring_job = types.model_deployment_monitoring_job_v1beta1, - types.model_monitoring = types.model_monitoring_v1beta1, + types.model_deployment_monitoring_job = ( + types.model_deployment_monitoring_job_v1beta1, + ) + types.model_monitoring = (types.model_monitoring_v1beta1,) types.model_service = types.model_service_v1beta1 types.operation = types.operation_v1beta1 types.pipeline_job = types.pipeline_job_v1beta1 @@ -176,8 +178,8 @@ types.model = types.model_v1 types.model_evaluation = types.model_evaluation_v1 types.model_evaluation_slice = types.model_evaluation_slice_v1 - types.model_deployment_monitoring_job = types.model_deployment_monitoring_job_v1, - types.model_monitoring = types.model_monitoring_v1, + types.model_deployment_monitoring_job = (types.model_deployment_monitoring_job_v1,) + types.model_monitoring = (types.model_monitoring_v1,) types.model_service = types.model_service_v1 types.operation = types.operation_v1 types.pipeline_job = types.pipeline_job_v1 diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 0e95ec64fd..b558e6e8cb 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -42,7 +42,6 @@ hyperparameter_tuning_job as gca_hyperparameter_tuning_job_compat, machine_resources as gca_machine_resources_compat, manual_batch_tuning_parameters as gca_manual_batch_tuning_parameters_compat, - study as gca_study_compat, model_deployment_monitoring_job as gca_model_deployment_monitoring_job_compat, model_monitoring as gca_model_monitoring_compat, @@ -384,7 +383,6 @@ def create( sync: bool = True, create_request_timeout: Optional[float] = None, batch_size: Optional[int] = None, - ) -> "BatchPredictionJob": """Create a batch prediction job. @@ -671,7 +669,7 @@ def create( gapic_batch_prediction_job.manual_batch_tuning_parameters = ( manual_batch_tuning_parameters ) - + # User Labels gapic_batch_prediction_job.labels = labels @@ -1923,7 +1921,7 @@ def trials(self) -> List[gca_study_compat.Trial]: class ModelDeploymentMonitoringJob(_Job): """Vertex AI Model Deployment Monitoring Job. - + This class should be used in conjunction with the Endpoint class in order to configure model monitoring for deployed models. """ @@ -1952,10 +1950,12 @@ def __init__( credentials=credentials, ) - - def _parse_configs(objective_configs: Union[ + def _parse_configs( + objective_configs: Union[ model_monitoring.EndpointObjectiveConfig, - Dict[str, model_monitoring.EndpointObjectiveConfig]]): + Dict[str, model_monitoring.EndpointObjectiveConfig], + ] + ): all_configs = {} all_models = [] @@ -1967,28 +1967,33 @@ def _parse_configs(objective_configs: Union[ all_models.append(model) ## when same objective config is applied to ALL models - if isinstance(objective_configs, model_monitoring.EndpointObjectiveConfig) and deployed_model_ids is None: + if ( + isinstance(objective_configs, model_monitoring.EndpointObjectiveConfig) + and deployed_model_ids is None + ): for model in all_models: all_configs[model] = objective_configs ## when same objective config is applied to SOME models - elif isinstance(objective_configs, model_monitoring.EndpointObjectiveConfig) and isinstance(deployed_model_ids, List): + elif isinstance( + objective_configs, model_monitoring.EndpointObjectiveConfig + ) and isinstance(deployed_model_ids, List): for model in deployed_model_ids: - assert(model in all_models) + assert model in all_models all_configs[model] = objective_configs ## when different objective configs are applied to EACH model elif isinstance(objective_configs, Dict) and deployed_model_ids is None: - assert(all(model in all_models for model in objective_configs.keys())) + assert all(model in all_models for model in objective_configs.keys()) all_configs = objective_configs - + mdm_objective_config_seq = [] for key in all_configs.keys(): mdm_objective_config_seq.append( gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringObjectiveConfig( - deployed_model_id = key, - objective_config = all_configs[key] - )) + deployed_model_id=key, objective_config=all_configs[key] + ) + ) return mdm_objective_config_seq @classmethod @@ -1998,7 +2003,8 @@ def create( endpoint: Union[str, "models.Endpoint"], objective_configs: Union[ model_monitoring.EndpointObjectiveConfig, - Dict[str, model_monitoring.EndpointObjectiveConfig]], + Dict[str, model_monitoring.EndpointObjectiveConfig], + ], logging_sampling_strategy: model_monitoring.RandomSampleConfig, monitor_interval: int, schedule_config: model_monitoring.ScheduleConfig, @@ -2057,7 +2063,7 @@ def create( schedule_config (model_monitoring.schedule.ScheduleConfig): Configures model monitoring job scheduling interval in hours. This defines how often the monitoring jobs are triggered. - + alert_config (model_monitoring.alert.EmailAlertConfig): Optional. Configures how alerts are sent to the user. Right now only email alert is supported. @@ -2111,7 +2117,7 @@ def create( underscores and dashes. International characters are allowed. See https://goo.gl/xmQnxf for more information and examples of labels. - + encryption_spec_key_name (str): Optional. Customer-managed encryption key spec for a ModelDeploymentMonitoringJob. If set, this @@ -2142,49 +2148,53 @@ def create( if stats_anomalies_base_directory: stats_anomalies_base_directory = gca_io_compat.GcsDestination( - output_uri_prefix = stats_anomalies_base_directory) - + output_uri_prefix=stats_anomalies_base_directory + ) + if encryption_spec_key_name: encryption_spec_key_name = gca_encryption_spec_compat.EncryptionSpec( - kms_key_name = encryption_spec_key_name) + kms_key_name=encryption_spec_key_name + ) mdm_objective_config_seq = cls._parse_configs(objective_configs) - self._gca_resource = gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringJob( - name=self.model_deployment_monitoring_job_name, - display_name=display_name, - endpoint=endpoint, - model_deployment_monitoring_objective_configs=mdm_objective_config_seq, - logging_sampling_strategy=logging_sampling_strategy, - model_deployment_monitoring_schedule_config=schedule_config, - model_monitoring_alert_config=alerting_config, - predict_instance_schema_uri=predict_instance_schema_uri, - analysis_instance_schema_uri=analysis_instance_schema_uri, - sample_predict_instance = sample_predict_instance, - stats_anomalies_base_directory = stats_anomalies_base_directory, - enable_monitoring_pipeline_logs = enable_monitoring_pipeline_logs, - labels = labels, - encryption_spec = encryption_spec_key_name + self._gca_resource = ( + gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringJob( + name=self.model_deployment_monitoring_job_name, + display_name=display_name, + endpoint=endpoint, + model_deployment_monitoring_objective_configs=mdm_objective_config_seq, + logging_sampling_strategy=logging_sampling_strategy, + model_deployment_monitoring_schedule_config=schedule_config, + model_monitoring_alert_config=alerting_config, + predict_instance_schema_uri=predict_instance_schema_uri, + analysis_instance_schema_uri=analysis_instance_schema_uri, + sample_predict_instance=sample_predict_instance, + stats_anomalies_base_directory=stats_anomalies_base_directory, + enable_monitoring_pipeline_logs=enable_monitoring_pipeline_logs, + labels=labels, + encryption_spec=encryption_spec_key_name, + ) ) api_client = cls.api_client mdm_job = api_client.create_model_deployment_monitoring_job( - parent = parent, - model_deployment_monitoring_job = cls._gca_resource + parent=parent, model_deployment_monitoring_job=cls._gca_resource ) return mdm_job - def update( self, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), display_name: Optional[str] = None, - objective_configs: Optional[Union[ - model_monitoring.EndpointObjectiveConfig, - Dict[str, model_monitoring.EndpointObjectiveConfig]]] = None, - logging_sampling_strategy: \ - Optional[model_monitoring.RandomSampleConfig] = None, + objective_configs: Optional[ + Union[ + model_monitoring.EndpointObjectiveConfig, + Dict[str, model_monitoring.EndpointObjectiveConfig], + ] + ] = None, + logging_sampling_strategy: Optional[model_monitoring.RandomSampleConfig] = None, monitor_interval: Optional[int] = None, deployed_model_ids: Optional[List[str]] = None, schedule_config: Optional[model_monitoring.ScheduleConfig] = None, @@ -2200,11 +2210,12 @@ def update( project: Optional[str] = None, location: Optional[str] = None, credentials: Optional[auth_credentials.Credentials] = None, - sync: bool = True, + sync: bool = True, ) -> "ModelDeploymentMonitoringJob": """""" current_job = self.api_client.get_model_deployment_monitoring_job( - name = slef.model_deployment_monitoring_job_name) + name=slef.model_deployment_monitoring_job_name + ) update_mask: List[str] = [] if display_name: update_mask.append("display_name") @@ -2226,27 +2237,32 @@ def update( current_job.log_ttl = bigquery_tables_log_ttl if enable_monitoring_pipeline_logs: update_mask.append("enable_monitoring_pipeline_logs") - current_job.enable_monitoring_pipeline_logs = enable_monitoring_pipeline_logs + current_job.enable_monitoring_pipeline_logs = ( + enable_monitoring_pipeline_logs + ) if objective_configs: update_mask.append("model_deployment_monitoring_objective_configs") - current_job.model_deployment_monitoring_objective_configs = self._parse_configs(objective_configs) + current_job.model_deployment_monitoring_objective_configs = ( + self._parse_configs(objective_configs) + ) self.api_client.update_model_deployment_monitoring_job( - model_deployment_monitoring_job = current_job, - update_mask = update_mask + model_deployment_monitoring_job=current_job, update_mask=update_mask ) - def pause(self) -> "ModelDeploymentMonitoringJob": """""" self.api_client.pause_model_deployment_monitoring_job( - self.model_deployment_monitoring_job_name) + self.model_deployment_monitoring_job_name + ) def resume(self) -> "ModelDeploymentMonitoringJob": """""" self.api_client.resume_model_deployment_monitoring_job( - self.model_deployment_monitoring_job_name) + self.model_deployment_monitoring_job_name + ) def delete(self) -> "ModelDeploymentMonitoringJob": """""" self.api_client.delete_model_deployment_monitoring_job( - self.model_deployment_monitoring_job_name) + self.model_deployment_monitoring_job_name + ) diff --git a/google/cloud/aiplatform/model_monitoring/__init__.py b/google/cloud/aiplatform/model_monitoring/__init__.py index cf86822ada..2935ff12cb 100644 --- a/google/cloud/aiplatform/model_monitoring/__init__.py +++ b/google/cloud/aiplatform/model_monitoring/__init__.py @@ -16,7 +16,12 @@ # from google.cloud.aiplatform.model_monitoring.alert import EmailAlertConfig -from google.cloud.aiplatform.model_monitoring.objective import EndpointSkewDetectionConfig, EndpointDriftDetectionConfig, EndpointExplanationConfig, EndpointObjectiveConfig +from google.cloud.aiplatform.model_monitoring.objective import ( + EndpointSkewDetectionConfig, + EndpointDriftDetectionConfig, + EndpointExplanationConfig, + EndpointObjectiveConfig, +) from google.cloud.aiplatform.model_monitoring.sampling import RandomSampleConfig from google.cloud.aiplatform.model_monitoring.schedule import ScheduleConfig @@ -27,5 +32,5 @@ "EndpointExplanationConfig", "EndpointObjectiveConfig", "RandomSampleConfig", - "ScheduleConfig" + "ScheduleConfig", ) diff --git a/google/cloud/aiplatform/model_monitoring/alert.py b/google/cloud/aiplatform/model_monitoring/alert.py index 5a39d88167..596acdce17 100644 --- a/google/cloud/aiplatform/model_monitoring/alert.py +++ b/google/cloud/aiplatform/model_monitoring/alert.py @@ -17,23 +17,20 @@ import abc from typing import Optional, List -from google.cloud.aiplatform.compat.types import model_monitoring as gca_model_monitoring +from google.cloud.aiplatform.compat.types import ( + model_monitoring as gca_model_monitoring, +) + class _AlertConfig(abc.ABC): """An abstract class for setting model monitoring alert config""" - def __init__( - self, - enable_logging: Optional[bool] = None - ): + + def __init__(self, enable_logging: Optional[bool] = None): self.enable_logging = enable_logging - + class EmailAlertConfig(_AlertConfig): - def __init__( - self, - user_emails: List[str], - enable_logging: Optional[bool] = None - ): + def __init__(self, user_emails: List[str], enable_logging: Optional[bool] = None): """Initializer for EmailAlertConfig Args: @@ -49,16 +46,16 @@ def __init__( Returns: An instance of EmailAlertConfig """ - super().__init__( - enable_logging = self.enable_logging - ) + super().__init__(enable_logging=self.enable_logging) self.user_emails = user_emails - + def as_proto(self): - user_email_alert_config = gca_model_monitoring.ModelMonitoringAlertConfig.EmailAlertConfig( - user_emails = self.user_emails + user_email_alert_config = ( + gca_model_monitoring.ModelMonitoringAlertConfig.EmailAlertConfig( + user_emails=self.user_emails + ) ) return gca_model_monitoring.ModelMonitoringAlertConfig( - email_alert_config = user_email_alert_config, - enable_logging = self.enable_logging + email_alert_config=user_email_alert_config, + enable_logging=self.enable_logging, ) diff --git a/google/cloud/aiplatform/model_monitoring/objective.py b/google/cloud/aiplatform/model_monitoring/objective.py index d2e28454de..132309f597 100644 --- a/google/cloud/aiplatform/model_monitoring/objective.py +++ b/google/cloud/aiplatform/model_monitoring/objective.py @@ -17,9 +17,12 @@ import abc from typing import Optional, Dict -from google.cloud.aiplatform.compat.types import model_monitoring as gca_model_monitoring +from google.cloud.aiplatform.compat.types import ( + model_monitoring as gca_model_monitoring, +) from google.cloud.aiplatform_v1.types import ThresholdConfig as gca_threshold_config + class _SkewDetectionConfig(abc.ABC): def __init__( self, @@ -40,18 +43,23 @@ def as_proto(self): skew_thresholds_mapping = {} attribution_score_skew_thresholds_mapping = {} for key in self.skew_thresholds.keys(): - skew_threshold = gca_threshold_config.ThresholdConfig(value = self.skew_thresholds[key]) + skew_threshold = gca_threshold_config.ThresholdConfig( + value=self.skew_thresholds[key] + ) skew_thresholds_mapping[key] = skew_threshold for key in self.attribute_skew_thresholds.keys(): - attribution_score_skew_threshold = gca_threshold_config.ThresholdConfig(value = self.attribute_skew_thresholds[key]) - attribution_score_skew_thresholds_mapping[key] = attribution_score_skew_threshold + attribution_score_skew_threshold = gca_threshold_config.ThresholdConfig( + value=self.attribute_skew_thresholds[key] + ) + attribution_score_skew_thresholds_mapping[ + key + ] = attribution_score_skew_threshold return gca_model_monitoring.ModelMonitoringObjectiveConfig.TrainingPredictionSkewDetectionConfig( - skew_thresholds = skew_thresholds_mapping, - attribution_score_skew_thresholds = attribution_score_skew_thresholds_mapping + skew_thresholds=skew_thresholds_mapping, + attribution_score_skew_thresholds=attribution_score_skew_thresholds_mapping, ) - - + class _DriftDetectionConfig(abc.ABC): def __init__( self, @@ -65,14 +73,20 @@ def as_proto(self): drift_thresholds_mapping = {} attribution_score_drift_thresholds_mapping = {} for key in self.drift_thresholds.keys(): - drift_threshold = gca_threshold_config.ThresholdConfig(value = self.drift_thresholds[key]) + drift_threshold = gca_threshold_config.ThresholdConfig( + value=self.drift_thresholds[key] + ) drift_thresholds_mapping[key] = drift_threshold for key in self.attribute_drift_thresholds.keys(): - attribution_score_drift_threshold = gca_threshold_config.ThresholdConfig(value = self.attribute_drift_thresholds[key]) - attribution_score_drift_thresholds_mapping[key] = attribution_score_drift_threshold + attribution_score_drift_threshold = gca_threshold_config.ThresholdConfig( + value=self.attribute_drift_thresholds[key] + ) + attribution_score_drift_thresholds_mapping[ + key + ] = attribution_score_drift_threshold return gca_model_monitoring.ModelMonitoringObjectiveConfig.PredictionDriftDetectionConfig( - drift_thresholds = drift_thresholds_mapping, - attribution_score_drift_thresholds = attribution_score_drift_thresholds_mapping + drift_thresholds=drift_thresholds_mapping, + attribution_score_drift_thresholds=attribution_score_drift_thresholds_mapping, ) @@ -81,17 +95,19 @@ def __init__(self): self.enable_feature_attributes = False def as_proto(self): - return gca_model_monitoring.ModelMonitoringObjectiveConfig.ExplanationConfig(enable_feature_attributes = self.enable_feature_attributes) + return gca_model_monitoring.ModelMonitoringObjectiveConfig.ExplanationConfig( + enable_feature_attributes=self.enable_feature_attributes + ) + class _ObjectiveConfig(abc.ABC): def __init__( self, - skew_detection_config: \ - Optional["model_monitoring._SkewDetectionConfig"] = None, - drift_detection_config: \ - Optional["model_monitoring._DriftDetectionConfig"] = None, - explanation_config: \ - Optional["model_monitoring._ExplanationConfig"] = None, + skew_detection_config: Optional["model_monitoring._SkewDetectionConfig"] = None, + drift_detection_config: Optional[ + "model_monitoring._DriftDetectionConfig" + ] = None, + explanation_config: Optional["model_monitoring._ExplanationConfig"] = None, ): self.skew_detection_config = skew_detection_config self.drift_detection_config = drift_detection_config @@ -102,10 +118,10 @@ def as_proto(self): if skew_detection_config is not None: training_dataset = skew_detection_config.data_source return gca_model_monitoring.ModelMonitoringObjectiveConfig( - training_dataset = training_dataset, - training_prediction_skew_detection_config = self.skew_detection_config, - prediction_drift_detection_config = self.drift_detection_config, - explanation = self.explanation_config + training_dataset=training_dataset, + training_prediction_skew_detection_config=self.skew_detection_config, + prediction_drift_detection_config=self.drift_detection_config, + explanation=self.explanation_config, ) @@ -116,6 +132,7 @@ class EndpointSkewDetectionConfig(_SkewDetectionConfig): distribution than the data used during model training. Model performance can deteriorate when production data deviates from training data. """ + def __init__( self, data_source: str, @@ -125,7 +142,7 @@ def __init__( target_field: Optional[str] = None, ): """Initializer for EndpointSkewDetectionConfig - + Args: data_source (str): Path to training dataset. @@ -140,8 +157,8 @@ def __init__( attribute_skew_thresholds (Dict[str, float]): Optional. Key is the feature name and value is the - threshold. Feature attributions indicate how much - each feature in your model contributed to the + threshold. Feature attributions indicate how much + each feature in your model contributed to the predictions for each given instance. data_format (str): @@ -163,7 +180,14 @@ def __init__( Returns: An instance of EndpointSkewDetectionConfig """ - super().__init__(data_source, skew_threshold, attribute_skew_thresholds, data_format, target_field) + super().__init__( + data_source, + skew_threshold, + attribute_skew_thresholds, + data_format, + target_field, + ) + class EndpointDriftDetectionConfig(_DriftDetectionConfig): """A class that configures prediction drift detection for models deployed to an endpoint. @@ -173,6 +197,7 @@ class EndpointDriftDetectionConfig(_DriftDetectionConfig): If original training data is available, EndpointSkewDetectionConfig should be set instead. """ + def __init__( self, drift_thresholds: Optional[Dict[str, float]] = None, @@ -190,25 +215,34 @@ def __init__( """ super().__init__(drift_threshold, attribute_drift_thresholds) + class EndpointExplanationConfig(_ExplanationConfig): """A class that enables Vertex Explainable AI. - + Only applicable if the model has explanation_spec populated. """ + def __init__(self): super().__init__() self.enable_feature_attributes = True + class EndpointObjectiveConfig(_ObjectiveConfig): """A class that captures skew detection, drift detection, and explaination configs.""" + def __init__( self, skew_detection_config: Optional[ - "model_monitoring.EndpointSkewDetectionConfig"] = None, + "model_monitoring.EndpointSkewDetectionConfig" + ] = None, drift_detection_config: Optional[ - "model_monitoring.EndpointDriftDetectionConfig"] = None, - explanation_config: \ - Optional["model_monitoring.EndpointExplanationConfig"] = None, + "model_monitoring.EndpointDriftDetectionConfig" + ] = None, + explanation_config: Optional[ + "model_monitoring.EndpointExplanationConfig" + ] = None, ): """""" - super().__init__(skew_detection_config, drift_detection_config, explanation_config) \ No newline at end of file + super().__init__( + skew_detection_config, drift_detection_config, explanation_config + ) diff --git a/google/cloud/aiplatform/model_monitoring/sampling.py b/google/cloud/aiplatform/model_monitoring/sampling.py index 4630963114..a8a78b193e 100644 --- a/google/cloud/aiplatform/model_monitoring/sampling.py +++ b/google/cloud/aiplatform/model_monitoring/sampling.py @@ -17,31 +17,32 @@ import abc from typing import Optional -from google.cloud.aiplatform.compat.types import model_monitoring as gca_model_monitoring +from google.cloud.aiplatform.compat.types import ( + model_monitoring as gca_model_monitoring, +) + class _SamplingStrategy(abc.ABC): """An abstract class for setting sampling strategy for model monitoring""" + class RandomSampleConfig(_SamplingStrategy): - def __init__( - self, - sample_rate: Optional[float] = None - ): + def __init__(self, sample_rate: Optional[float] = None): """Initializer for RandomSampleConfig - Args: - sample_rate (float): - Optional. Sets the sampling rate for model monitoring logs. - If not set, all logs are processed. - Returns: - An instance of RandomSampleConfig + Args: + sample_rate (float): + Optional. Sets the sampling rate for model monitoring logs. + If not set, all logs are processed. + Returns: + An instance of RandomSampleConfig """ super().__init() self.sample_rate = sample_rate def as_proto(self): return gca_model_monitoring.SamplingStrategy( - random_sample_config = gca_model_monitoring.SamplingStrategy.RandomSampleConfig( - sample_rate = self.sample_rate + random_sample_config=gca_model_monitoring.SamplingStrategy.RandomSampleConfig( + sample_rate=self.sample_rate ) ) diff --git a/google/cloud/aiplatform/model_monitoring/schedule.py b/google/cloud/aiplatform/model_monitoring/schedule.py index b761d5bf15..75ea80b0e4 100644 --- a/google/cloud/aiplatform/model_monitoring/schedule.py +++ b/google/cloud/aiplatform/model_monitoring/schedule.py @@ -17,30 +17,34 @@ import abc from google.protobuf import duration_pb2 # type: ignore -from google.cloud.aiplatform.compat.types import model_deployment_monitoring_job as gca_model_deployment_monitoring_job +from google.cloud.aiplatform.compat.types import ( + model_deployment_monitoring_job as gca_model_deployment_monitoring_job, +) + class _ScheduleConfig(abc.ABC): """""" + class ScheduleConfig(_ScheduleConfig): """""" - def __init__( - self, - monitor_interval: int - ): + + def __init__(self, monitor_interval: int): """Initializer for ScheduleConfig - Args: - monitor_interval (int): - Sets the model monitoring job scheduling interval in hours. - This defines how often the monitoring jobs are triggered. - Returns: - An instance of ScheduleConfig + Args: + monitor_interval (int): + Sets the model monitoring job scheduling interval in hours. + This defines how often the monitoring jobs are triggered. + Returns: + An instance of ScheduleConfig """ super().__init__() self.monitor_interval = monitor_interval def as_proto(self): - return gca_model_deployment_monitoring_job.ModelDeploymentMonitoringScheduleConfig( - monitor_interval = duration_pb2.Duration(seconds = self.monitor_interval) + return ( + gca_model_deployment_monitoring_job.ModelDeploymentMonitoringScheduleConfig( + monitor_interval=duration_pb2.Duration(seconds=self.monitor_interval) + ) ) From 1b6178f22a0dfb43869409beb57bb0dcbdc7e873 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Thu, 26 May 2022 10:33:32 -0700 Subject: [PATCH 04/53] resolving merge diff --- google/cloud/aiplatform/jobs.py | 1 - google/cloud/aiplatform/model_monitoring/alert.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index b558e6e8cb..43e2f22d18 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -2063,7 +2063,6 @@ def create( schedule_config (model_monitoring.schedule.ScheduleConfig): Configures model monitoring job scheduling interval in hours. This defines how often the monitoring jobs are triggered. - alert_config (model_monitoring.alert.EmailAlertConfig): Optional. Configures how alerts are sent to the user. Right now only email alert is supported. diff --git a/google/cloud/aiplatform/model_monitoring/alert.py b/google/cloud/aiplatform/model_monitoring/alert.py index 596acdce17..573c7b8f1f 100644 --- a/google/cloud/aiplatform/model_monitoring/alert.py +++ b/google/cloud/aiplatform/model_monitoring/alert.py @@ -21,14 +21,12 @@ model_monitoring as gca_model_monitoring, ) - class _AlertConfig(abc.ABC): """An abstract class for setting model monitoring alert config""" def __init__(self, enable_logging: Optional[bool] = None): self.enable_logging = enable_logging - class EmailAlertConfig(_AlertConfig): def __init__(self, user_emails: List[str], enable_logging: Optional[bool] = None): """Initializer for EmailAlertConfig From 7eb38797d4c20f18638d75186a879e4c9ac2b5bc Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Thu, 26 May 2022 17:36:47 +0000 Subject: [PATCH 05/53] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- google/cloud/aiplatform/model_monitoring/alert.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/google/cloud/aiplatform/model_monitoring/alert.py b/google/cloud/aiplatform/model_monitoring/alert.py index 573c7b8f1f..596acdce17 100644 --- a/google/cloud/aiplatform/model_monitoring/alert.py +++ b/google/cloud/aiplatform/model_monitoring/alert.py @@ -21,12 +21,14 @@ model_monitoring as gca_model_monitoring, ) + class _AlertConfig(abc.ABC): """An abstract class for setting model monitoring alert config""" def __init__(self, enable_logging: Optional[bool] = None): self.enable_logging = enable_logging + class EmailAlertConfig(_AlertConfig): def __init__(self, user_emails: List[str], enable_logging: Optional[bool] = None): """Initializer for EmailAlertConfig From 161abf23066252b393e5117d85b0c1c2b249ea4a Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Sat, 28 May 2022 20:53:46 -0700 Subject: [PATCH 06/53] removing sync parameter from MDM job --- google/cloud/aiplatform/jobs.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 43e2f22d18..e0a2f870ef 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -2010,7 +2010,6 @@ def create( schedule_config: model_monitoring.ScheduleConfig, metadata: Sequence[Tuple[str, str]], timeout: float = None, - sync: bool = True, deployed_model_ids: Optional[List[str]] = ["*"], alert_config: Optional[model_monitoring.EmailAlertConfig] = None, predict_instance_schema_uri: Optional[str] = None, @@ -2127,11 +2126,9 @@ def create( timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), - project: Optional[str] = None, location: Optional[str] = None, credentials: Optional[auth_credentials.Credentials] = None, - sync: bool = True Returns: An instance of ModelDeploymentMonitoringJob @@ -2178,7 +2175,9 @@ def create( api_client = cls.api_client mdm_job = api_client.create_model_deployment_monitoring_job( - parent=parent, model_deployment_monitoring_job=cls._gca_resource + parent=parent, + model_deployment_monitoring_job=cls._gca_resource, + timeout = timeout ) return mdm_job From 4b4695065c77861b4826fd3b7a4abed9cf39fa85 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Sun, 29 May 2022 03:56:27 +0000 Subject: [PATCH 07/53] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- google/cloud/aiplatform/jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index e0a2f870ef..1bc0049a4c 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -2177,7 +2177,7 @@ def create( mdm_job = api_client.create_model_deployment_monitoring_job( parent=parent, model_deployment_monitoring_job=cls._gca_resource, - timeout = timeout + timeout=timeout, ) return mdm_job From 658e18e2c7a8e92e671f4cf4665ecbd8f52ca737 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Wed, 1 Jun 2022 00:30:53 -0700 Subject: [PATCH 08/53] fixing runtime errors --- google/cloud/aiplatform/model_monitoring/alert.py | 2 +- google/cloud/aiplatform/model_monitoring/sampling.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/google/cloud/aiplatform/model_monitoring/alert.py b/google/cloud/aiplatform/model_monitoring/alert.py index 596acdce17..0feb5c3735 100644 --- a/google/cloud/aiplatform/model_monitoring/alert.py +++ b/google/cloud/aiplatform/model_monitoring/alert.py @@ -46,7 +46,7 @@ def __init__(self, user_emails: List[str], enable_logging: Optional[bool] = None Returns: An instance of EmailAlertConfig """ - super().__init__(enable_logging=self.enable_logging) + super().__init__(enable_logging=enable_logging) self.user_emails = user_emails def as_proto(self): diff --git a/google/cloud/aiplatform/model_monitoring/sampling.py b/google/cloud/aiplatform/model_monitoring/sampling.py index a8a78b193e..0794eda429 100644 --- a/google/cloud/aiplatform/model_monitoring/sampling.py +++ b/google/cloud/aiplatform/model_monitoring/sampling.py @@ -37,7 +37,7 @@ def __init__(self, sample_rate: Optional[float] = None): Returns: An instance of RandomSampleConfig """ - super().__init() + super().__init__() self.sample_rate = sample_rate def as_proto(self): From 79ea3ed7ca7506c1cd098a24ff332b6b7a87ad84 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Mon, 6 Jun 2022 22:36:16 -0700 Subject: [PATCH 09/53] fixing more runtime errors --- google/cloud/aiplatform/__init__.py | 1 + google/cloud/aiplatform/jobs.py | 192 ++++++++++++------ .../aiplatform/model_monitoring/alert.py | 2 +- .../aiplatform/model_monitoring/objective.py | 47 +++-- .../aiplatform/model_monitoring/sampling.py | 2 +- .../aiplatform/model_monitoring/schedule.py | 2 +- 6 files changed, 169 insertions(+), 77 deletions(-) diff --git a/google/cloud/aiplatform/__init__.py b/google/cloud/aiplatform/__init__.py index db7d0a7c18..f90ab5f18d 100644 --- a/google/cloud/aiplatform/__init__.py +++ b/google/cloud/aiplatform/__init__.py @@ -50,6 +50,7 @@ BatchPredictionJob, CustomJob, HyperparameterTuningJob, + ModelDeploymentMonitoringJob ) from google.cloud.aiplatform.pipeline_jobs import PipelineJob from google.cloud.aiplatform.tensorboard import ( diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 1bc0049a4c..dde9b860b7 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -43,8 +43,10 @@ machine_resources as gca_machine_resources_compat, manual_batch_tuning_parameters as gca_manual_batch_tuning_parameters_compat, study as gca_study_compat, - model_deployment_monitoring_job as gca_model_deployment_monitoring_job_compat, - model_monitoring as gca_model_monitoring_compat, +) + +from google.cloud.aiplatform_v1.types import ( + model_deployment_monitoring_job as gca_model_deployment_monitoring_job ) from google.cloud.aiplatform.constants import base as constants from google.cloud.aiplatform import initializer @@ -1949,52 +1951,100 @@ def __init__( location=location, credentials=credentials, ) + self._gca_resource = self._get_gca_resource(resource_name=job_name) + + @classmethod def _parse_configs( + cls, objective_configs: Union[ model_monitoring.EndpointObjectiveConfig, Dict[str, model_monitoring.EndpointObjectiveConfig], - ] + ], + project, + location, + endpoint, + deployed_model_ids ): - - all_configs = {} + all_configs = [] all_models = [] - client_options = dict(api_endpoint=API_ENDPOINT) - client = endpoint_service_client(client_options=client_options) - parent = f"projects/{self.project}/locations/{self.location}" + default_endpoint = "aiplatform.googleapis.com" + client_options = dict(api_endpoint=f"{location}-{default_endpoint}") + client = endpoint_service_client.EndpointServiceClient(client_options=client_options) + parent = f"projects/{project}/locations/{location}" response = client.get_endpoint(name=f"{parent}/endpoints/{endpoint}") for model in response.deployed_models: - all_models.append(model) + all_models.append(model.id) ## when same objective config is applied to ALL models if ( isinstance(objective_configs, model_monitoring.EndpointObjectiveConfig) - and deployed_model_ids is None + and '*' in deployed_model_ids ): for model in all_models: - all_configs[model] = objective_configs + all_configs.append( + gca_model_deployment_monitoring_job.ModelDeploymentMonitoringObjectiveConfig( + deployed_model_id=model, objective_config=objective_configs.as_proto() + ) + ) ## when same objective config is applied to SOME models - elif isinstance( + elif (isinstance( objective_configs, model_monitoring.EndpointObjectiveConfig - ) and isinstance(deployed_model_ids, List): + ) and '*' not in deployed_model_ids): for model in deployed_model_ids: assert model in all_models - all_configs[model] = objective_configs + all_configs.append( + gca_model_deployment_monitoring_job.ModelDeploymentMonitoringObjectiveConfig( + deployed_model_id=model, objective_config=objective_configs.as_proto() + ) + ) ## when different objective configs are applied to EACH model elif isinstance(objective_configs, Dict) and deployed_model_ids is None: assert all(model in all_models for model in objective_configs.keys()) - all_configs = objective_configs - - mdm_objective_config_seq = [] - for key in all_configs.keys(): - mdm_objective_config_seq.append( - gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringObjectiveConfig( - deployed_model_id=key, objective_config=all_configs[key] + for key in objective_configs.keys(): + all_configs.append( + gca_model_deployment_monitoring_job.ModelDeploymentMonitoringObjectiveConfig( + deployed_model_id=key, objective_config=objective_configs[key].as_proto() + ) ) - ) - return mdm_objective_config_seq + + print(all_configs) + return all_configs + + @classmethod + def _empty_constructor( + cls, + project: Optional[str] = None, + location: Optional[str] = None, + credentials: Optional[auth_credentials.Credentials] = None, + resource_name: Optional[str] = None, + ) -> "_Job": + """Initializes with all attributes set to None. + + The attributes should be populated after a future is complete. This allows + scheduling of additional API calls before the resource is created. + + Args: + project (str): Optional. Project of the resource noun. + location (str): Optional. The location of the resource noun. + credentials(google.auth.credentials.Credentials): + Optional. custom credentials to use when accessing interacting with + resource noun. + resource_name(str): Optional. A fully-qualified resource name or ID. + Returns: + An instance of this class with attributes set to None. + """ + self = super()._empty_constructor( + project=project, + location=location, + credentials=credentials, + resource_name=resource_name, + ) + + self._logged_web_access_uris = set() + return self @classmethod def create( @@ -2008,9 +2058,9 @@ def create( logging_sampling_strategy: model_monitoring.RandomSampleConfig, monitor_interval: int, schedule_config: model_monitoring.ScheduleConfig, - metadata: Sequence[Tuple[str, str]], timeout: float = None, deployed_model_ids: Optional[List[str]] = ["*"], + metadata: Optional[Sequence[Tuple[str, str]]] = None, alert_config: Optional[model_monitoring.EmailAlertConfig] = None, predict_instance_schema_uri: Optional[str] = None, sample_predict_instance: Optional[str] = None, @@ -2152,63 +2202,83 @@ def create( kms_key_name=encryption_spec_key_name ) - mdm_objective_config_seq = cls._parse_configs(objective_configs) + mdm_objective_config_seq = cls._parse_configs(objective_configs, project, location, endpoint, deployed_model_ids) - self._gca_resource = ( - gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringJob( - name=self.model_deployment_monitoring_job_name, - display_name=display_name, - endpoint=endpoint, - model_deployment_monitoring_objective_configs=mdm_objective_config_seq, - logging_sampling_strategy=logging_sampling_strategy, - model_deployment_monitoring_schedule_config=schedule_config, - model_monitoring_alert_config=alerting_config, - predict_instance_schema_uri=predict_instance_schema_uri, - analysis_instance_schema_uri=analysis_instance_schema_uri, - sample_predict_instance=sample_predict_instance, - stats_anomalies_base_directory=stats_anomalies_base_directory, - enable_monitoring_pipeline_logs=enable_monitoring_pipeline_logs, - labels=labels, - encryption_spec=encryption_spec_key_name, - ) + empty_mdm_job = cls._empty_constructor( + project = project, + location = location, + credentials = credentials, ) - api_client = cls.api_client - mdm_job = api_client.create_model_deployment_monitoring_job( + parent = initializer.global_config.common_location_path( + project=empty_mdm_job.project, + location=empty_mdm_job.location, + ) + + api_client = empty_mdm_job.api_client + + _LOGGER.log_create_with_lro(cls) + + if int(endpoint): + formatted_endpoint = f"projects/{project}/locations/{location}/endpoints/{endpoint}" + + gapic_mdm_job = gca_model_deployment_monitoring_job.ModelDeploymentMonitoringJob( + display_name=display_name, + endpoint=endpoint, + model_deployment_monitoring_objective_configs=mdm_objective_config_seq, + logging_sampling_strategy=logging_sampling_strategy.as_proto(), + model_deployment_monitoring_schedule_config=schedule_config.as_proto(), + model_monitoring_alert_config=alert_config.as_proto(), + predict_instance_schema_uri=predict_instance_schema_uri, + analysis_instance_schema_uri=analysis_instance_schema_uri, + sample_predict_instance=sample_predict_instance, + stats_anomalies_base_directory=stats_anomalies_base_directory, + enable_monitoring_pipeline_logs=enable_monitoring_pipeline_logs, + labels=labels, + encryption_spec=encryption_spec_key_name, + ) + + gca_mdm_job = api_client.create_model_deployment_monitoring_job( parent=parent, - model_deployment_monitoring_job=cls._gca_resource, + model_deployment_monitoring_job=gapic_mdm_job, timeout=timeout, ) + gca_mdm_job._gca_resource = gapic_mdm_job + + empty_mdm_job._gca_resource = gca_mdm_job + + mdm_job = empty_mdm_job + + _LOGGER.log_create_complete(cls, mdm_job._gca_resource, "mdm") + + _LOGGER.info( + "View Model Deployment Monitoring Job:\n%s" % mdm_job._dashboard_uri() + ) + return mdm_job + def update( self, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), display_name: Optional[str] = None, + schedule_config: Optional[model_monitoring.ScheduleConfig] = None, + alert_config: Optional[model_monitoring.EmailAlertConfig] = None, + logging_sampling_strategy: Optional[model_monitoring.RandomSampleConfig] = None, + labels: Optional[Dict[str, str]] = None, + bigquery_tables_log_ttl: Optional[int] = None, + enable_monitoring_pipeline_logs: Optional[bool] = None, objective_configs: Optional[ Union[ model_monitoring.EndpointObjectiveConfig, Dict[str, model_monitoring.EndpointObjectiveConfig], ] ] = None, - logging_sampling_strategy: Optional[model_monitoring.RandomSampleConfig] = None, - monitor_interval: Optional[int] = None, deployed_model_ids: Optional[List[str]] = None, - schedule_config: Optional[model_monitoring.ScheduleConfig] = None, - alert_config: Optional[model_monitoring.EmailAlertConfig] = None, - predict_instance_schema_uri: Optional[str] = None, - sample_predict_instance: Optional[str] = None, - analysis_instance_schema_uri: Optional[str] = None, - bigquery_tables_log_ttl: Optional[int] = None, - stats_anomalies_base_directory: Optional[str] = None, - enable_monitoring_pipeline_logs: Optional[bool] = None, - labels: Optional[Dict[str, str]] = None, - encryption_spec_key_name: Optional[str] = None, - project: Optional[str] = None, - location: Optional[str] = None, - credentials: Optional[auth_credentials.Credentials] = None, - sync: bool = True, + training_dataset: Optional[str] = None, + training_prediction_skew_detection_config: Optional[model_monitoring.EndpointSkewDetectionConfig] = None, + prediction_drift_detection_config: Optional[model_monitoring.EndpointDriftDetectionConfig] = None, ) -> "ModelDeploymentMonitoringJob": """""" current_job = self.api_client.get_model_deployment_monitoring_job( diff --git a/google/cloud/aiplatform/model_monitoring/alert.py b/google/cloud/aiplatform/model_monitoring/alert.py index 0feb5c3735..d7390fafa5 100644 --- a/google/cloud/aiplatform/model_monitoring/alert.py +++ b/google/cloud/aiplatform/model_monitoring/alert.py @@ -17,7 +17,7 @@ import abc from typing import Optional, List -from google.cloud.aiplatform.compat.types import ( +from google.cloud.aiplatform_v1.types import ( model_monitoring as gca_model_monitoring, ) diff --git a/google/cloud/aiplatform/model_monitoring/objective.py b/google/cloud/aiplatform/model_monitoring/objective.py index 132309f597..21b95c41fb 100644 --- a/google/cloud/aiplatform/model_monitoring/objective.py +++ b/google/cloud/aiplatform/model_monitoring/objective.py @@ -17,10 +17,16 @@ import abc from typing import Optional, Dict -from google.cloud.aiplatform.compat.types import ( +# from google.cloud.aiplatform.compat.types import ( +# model_monitoring as gca_model_monitoring, +# ) +# from google.cloud.aiplatform_v1.types import ThresholdConfig as gca_threshold_config +# from google.cloud.aiplatform_v1.types.io import BigQuerySource +from google.cloud.aiplatform_v1.types import ( + io as gca_io, + ThresholdConfig as gca_threshold_config, model_monitoring as gca_model_monitoring, ) -from google.cloud.aiplatform_v1.types import ThresholdConfig as gca_threshold_config class _SkewDetectionConfig(abc.ABC): @@ -33,6 +39,7 @@ def __init__( target_field: Optional[str] = None, ): """""" + # print(skew_thresholds) self.data_source = data_source self.skew_thresholds = skew_thresholds self.attribute_skew_thresholds = attribute_skew_thresholds @@ -43,12 +50,12 @@ def as_proto(self): skew_thresholds_mapping = {} attribution_score_skew_thresholds_mapping = {} for key in self.skew_thresholds.keys(): - skew_threshold = gca_threshold_config.ThresholdConfig( + skew_threshold = gca_threshold_config( value=self.skew_thresholds[key] ) skew_thresholds_mapping[key] = skew_threshold for key in self.attribute_skew_thresholds.keys(): - attribution_score_skew_threshold = gca_threshold_config.ThresholdConfig( + attribution_score_skew_threshold = gca_threshold_config( value=self.attribute_skew_thresholds[key] ) attribution_score_skew_thresholds_mapping[ @@ -73,12 +80,12 @@ def as_proto(self): drift_thresholds_mapping = {} attribution_score_drift_thresholds_mapping = {} for key in self.drift_thresholds.keys(): - drift_threshold = gca_threshold_config.ThresholdConfig( + drift_threshold = gca_threshold_config( value=self.drift_thresholds[key] ) drift_thresholds_mapping[key] = drift_threshold for key in self.attribute_drift_thresholds.keys(): - attribution_score_drift_threshold = gca_threshold_config.ThresholdConfig( + attribution_score_drift_threshold = gca_threshold_config( value=self.attribute_drift_thresholds[key] ) attribution_score_drift_thresholds_mapping[ @@ -115,13 +122,27 @@ def __init__( def as_proto(self): training_dataset = None - if skew_detection_config is not None: - training_dataset = skew_detection_config.data_source + # print(self.skew_detection_config.target_field) + if self.skew_detection_config is not None: + training_dataset = gca_model_monitoring.ModelMonitoringObjectiveConfig.TrainingDataset( + target_field = self.skew_detection_config.target_field + ) + if 'bq:/' in self.skew_detection_config.data_source: + training_dataset.bigquery_source = gca_io.BigQuerySource( + input_uri = self.skew_detection_config.data_source + ) + elif 'gs:/' in self.skew_detection_config.data_source: + training_dataset.gcs_source = gca_io.GcsSource( + uris = [self.skew_detection_config.data_source] + ) + else: + training_dataset.dataset = self.skew_detection_config.data_source + # print(training_dataset) return gca_model_monitoring.ModelMonitoringObjectiveConfig( training_dataset=training_dataset, - training_prediction_skew_detection_config=self.skew_detection_config, - prediction_drift_detection_config=self.drift_detection_config, - explanation=self.explanation_config, + training_prediction_skew_detection_config=self.skew_detection_config.as_proto(), + prediction_drift_detection_config=self.drift_detection_config.as_proto(), + explanation_config=self.explanation_config.as_proto(), ) @@ -182,7 +203,7 @@ def __init__( """ super().__init__( data_source, - skew_threshold, + skew_thresholds, attribute_skew_thresholds, data_format, target_field, @@ -213,7 +234,7 @@ def __init__( Returns: An instance of EndpointDriftDetectionConfig """ - super().__init__(drift_threshold, attribute_drift_thresholds) + super().__init__(drift_thresholds, attribute_drift_thresholds) class EndpointExplanationConfig(_ExplanationConfig): diff --git a/google/cloud/aiplatform/model_monitoring/sampling.py b/google/cloud/aiplatform/model_monitoring/sampling.py index 0794eda429..89513c6ee7 100644 --- a/google/cloud/aiplatform/model_monitoring/sampling.py +++ b/google/cloud/aiplatform/model_monitoring/sampling.py @@ -17,7 +17,7 @@ import abc from typing import Optional -from google.cloud.aiplatform.compat.types import ( +from google.cloud.aiplatform_v1.types import ( model_monitoring as gca_model_monitoring, ) diff --git a/google/cloud/aiplatform/model_monitoring/schedule.py b/google/cloud/aiplatform/model_monitoring/schedule.py index 75ea80b0e4..c2f2389530 100644 --- a/google/cloud/aiplatform/model_monitoring/schedule.py +++ b/google/cloud/aiplatform/model_monitoring/schedule.py @@ -17,7 +17,7 @@ import abc from google.protobuf import duration_pb2 # type: ignore -from google.cloud.aiplatform.compat.types import ( +from google.cloud.aiplatform_v1.types import ( model_deployment_monitoring_job as gca_model_deployment_monitoring_job, ) From 2a30817a92130c628c64087615fc5d9628198706 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 7 Jun 2022 05:38:56 +0000 Subject: [PATCH 10/53] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- google/cloud/aiplatform/__init__.py | 2 +- google/cloud/aiplatform/jobs.py | 80 +++++++++++-------- .../aiplatform/model_monitoring/objective.py | 23 +++--- 3 files changed, 59 insertions(+), 46 deletions(-) diff --git a/google/cloud/aiplatform/__init__.py b/google/cloud/aiplatform/__init__.py index f90ab5f18d..4ca60ff30a 100644 --- a/google/cloud/aiplatform/__init__.py +++ b/google/cloud/aiplatform/__init__.py @@ -50,7 +50,7 @@ BatchPredictionJob, CustomJob, HyperparameterTuningJob, - ModelDeploymentMonitoringJob + ModelDeploymentMonitoringJob, ) from google.cloud.aiplatform.pipeline_jobs import PipelineJob from google.cloud.aiplatform.tensorboard import ( diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index dde9b860b7..7dfbf61965 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -46,7 +46,7 @@ ) from google.cloud.aiplatform_v1.types import ( - model_deployment_monitoring_job as gca_model_deployment_monitoring_job + model_deployment_monitoring_job as gca_model_deployment_monitoring_job, ) from google.cloud.aiplatform.constants import base as constants from google.cloud.aiplatform import initializer @@ -1953,7 +1953,6 @@ def __init__( ) self._gca_resource = self._get_gca_resource(resource_name=job_name) - @classmethod def _parse_configs( cls, @@ -1964,13 +1963,15 @@ def _parse_configs( project, location, endpoint, - deployed_model_ids + deployed_model_ids, ): all_configs = [] all_models = [] default_endpoint = "aiplatform.googleapis.com" client_options = dict(api_endpoint=f"{location}-{default_endpoint}") - client = endpoint_service_client.EndpointServiceClient(client_options=client_options) + client = endpoint_service_client.EndpointServiceClient( + client_options=client_options + ) parent = f"projects/{project}/locations/{location}" response = client.get_endpoint(name=f"{parent}/endpoints/{endpoint}") for model in response.deployed_models: @@ -1979,24 +1980,27 @@ def _parse_configs( ## when same objective config is applied to ALL models if ( isinstance(objective_configs, model_monitoring.EndpointObjectiveConfig) - and '*' in deployed_model_ids + and "*" in deployed_model_ids ): for model in all_models: all_configs.append( gca_model_deployment_monitoring_job.ModelDeploymentMonitoringObjectiveConfig( - deployed_model_id=model, objective_config=objective_configs.as_proto() + deployed_model_id=model, + objective_config=objective_configs.as_proto(), ) ) ## when same objective config is applied to SOME models - elif (isinstance( - objective_configs, model_monitoring.EndpointObjectiveConfig - ) and '*' not in deployed_model_ids): + elif ( + isinstance(objective_configs, model_monitoring.EndpointObjectiveConfig) + and "*" not in deployed_model_ids + ): for model in deployed_model_ids: assert model in all_models all_configs.append( gca_model_deployment_monitoring_job.ModelDeploymentMonitoringObjectiveConfig( - deployed_model_id=model, objective_config=objective_configs.as_proto() + deployed_model_id=model, + objective_config=objective_configs.as_proto(), ) ) @@ -2006,7 +2010,8 @@ def _parse_configs( for key in objective_configs.keys(): all_configs.append( gca_model_deployment_monitoring_job.ModelDeploymentMonitoringObjectiveConfig( - deployed_model_id=key, objective_config=objective_configs[key].as_proto() + deployed_model_id=key, + objective_config=objective_configs[key].as_proto(), ) ) @@ -2202,12 +2207,14 @@ def create( kms_key_name=encryption_spec_key_name ) - mdm_objective_config_seq = cls._parse_configs(objective_configs, project, location, endpoint, deployed_model_ids) + mdm_objective_config_seq = cls._parse_configs( + objective_configs, project, location, endpoint, deployed_model_ids + ) empty_mdm_job = cls._empty_constructor( - project = project, - location = location, - credentials = credentials, + project=project, + location=location, + credentials=credentials, ) parent = initializer.global_config.common_location_path( @@ -2220,22 +2227,26 @@ def create( _LOGGER.log_create_with_lro(cls) if int(endpoint): - formatted_endpoint = f"projects/{project}/locations/{location}/endpoints/{endpoint}" + formatted_endpoint = ( + f"projects/{project}/locations/{location}/endpoints/{endpoint}" + ) - gapic_mdm_job = gca_model_deployment_monitoring_job.ModelDeploymentMonitoringJob( - display_name=display_name, - endpoint=endpoint, - model_deployment_monitoring_objective_configs=mdm_objective_config_seq, - logging_sampling_strategy=logging_sampling_strategy.as_proto(), - model_deployment_monitoring_schedule_config=schedule_config.as_proto(), - model_monitoring_alert_config=alert_config.as_proto(), - predict_instance_schema_uri=predict_instance_schema_uri, - analysis_instance_schema_uri=analysis_instance_schema_uri, - sample_predict_instance=sample_predict_instance, - stats_anomalies_base_directory=stats_anomalies_base_directory, - enable_monitoring_pipeline_logs=enable_monitoring_pipeline_logs, - labels=labels, - encryption_spec=encryption_spec_key_name, + gapic_mdm_job = ( + gca_model_deployment_monitoring_job.ModelDeploymentMonitoringJob( + display_name=display_name, + endpoint=endpoint, + model_deployment_monitoring_objective_configs=mdm_objective_config_seq, + logging_sampling_strategy=logging_sampling_strategy.as_proto(), + model_deployment_monitoring_schedule_config=schedule_config.as_proto(), + model_monitoring_alert_config=alert_config.as_proto(), + predict_instance_schema_uri=predict_instance_schema_uri, + analysis_instance_schema_uri=analysis_instance_schema_uri, + sample_predict_instance=sample_predict_instance, + stats_anomalies_base_directory=stats_anomalies_base_directory, + enable_monitoring_pipeline_logs=enable_monitoring_pipeline_logs, + labels=labels, + encryption_spec=encryption_spec_key_name, + ) ) gca_mdm_job = api_client.create_model_deployment_monitoring_job( @@ -2257,7 +2268,6 @@ def create( return mdm_job - def update( self, timeout: float = None, @@ -2277,8 +2287,12 @@ def update( ] = None, deployed_model_ids: Optional[List[str]] = None, training_dataset: Optional[str] = None, - training_prediction_skew_detection_config: Optional[model_monitoring.EndpointSkewDetectionConfig] = None, - prediction_drift_detection_config: Optional[model_monitoring.EndpointDriftDetectionConfig] = None, + training_prediction_skew_detection_config: Optional[ + model_monitoring.EndpointSkewDetectionConfig + ] = None, + prediction_drift_detection_config: Optional[ + model_monitoring.EndpointDriftDetectionConfig + ] = None, ) -> "ModelDeploymentMonitoringJob": """""" current_job = self.api_client.get_model_deployment_monitoring_job( diff --git a/google/cloud/aiplatform/model_monitoring/objective.py b/google/cloud/aiplatform/model_monitoring/objective.py index 21b95c41fb..3a51da3cd7 100644 --- a/google/cloud/aiplatform/model_monitoring/objective.py +++ b/google/cloud/aiplatform/model_monitoring/objective.py @@ -17,6 +17,7 @@ import abc from typing import Optional, Dict + # from google.cloud.aiplatform.compat.types import ( # model_monitoring as gca_model_monitoring, # ) @@ -50,9 +51,7 @@ def as_proto(self): skew_thresholds_mapping = {} attribution_score_skew_thresholds_mapping = {} for key in self.skew_thresholds.keys(): - skew_threshold = gca_threshold_config( - value=self.skew_thresholds[key] - ) + skew_threshold = gca_threshold_config(value=self.skew_thresholds[key]) skew_thresholds_mapping[key] = skew_threshold for key in self.attribute_skew_thresholds.keys(): attribution_score_skew_threshold = gca_threshold_config( @@ -80,9 +79,7 @@ def as_proto(self): drift_thresholds_mapping = {} attribution_score_drift_thresholds_mapping = {} for key in self.drift_thresholds.keys(): - drift_threshold = gca_threshold_config( - value=self.drift_thresholds[key] - ) + drift_threshold = gca_threshold_config(value=self.drift_thresholds[key]) drift_thresholds_mapping[key] = drift_threshold for key in self.attribute_drift_thresholds.keys(): attribution_score_drift_threshold = gca_threshold_config( @@ -124,16 +121,18 @@ def as_proto(self): training_dataset = None # print(self.skew_detection_config.target_field) if self.skew_detection_config is not None: - training_dataset = gca_model_monitoring.ModelMonitoringObjectiveConfig.TrainingDataset( - target_field = self.skew_detection_config.target_field + training_dataset = ( + gca_model_monitoring.ModelMonitoringObjectiveConfig.TrainingDataset( + target_field=self.skew_detection_config.target_field + ) ) - if 'bq:/' in self.skew_detection_config.data_source: + if "bq:/" in self.skew_detection_config.data_source: training_dataset.bigquery_source = gca_io.BigQuerySource( - input_uri = self.skew_detection_config.data_source + input_uri=self.skew_detection_config.data_source ) - elif 'gs:/' in self.skew_detection_config.data_source: + elif "gs:/" in self.skew_detection_config.data_source: training_dataset.gcs_source = gca_io.GcsSource( - uris = [self.skew_detection_config.data_source] + uris=[self.skew_detection_config.data_source] ) else: training_dataset.dataset = self.skew_detection_config.data_source From 327611ca0e8596ed3683bfcef6307b49f1465736 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Tue, 7 Jun 2022 06:14:33 +0000 Subject: [PATCH 11/53] fixing some more linter errors --- google/cloud/aiplatform/__init__.py | 1 + google/cloud/aiplatform/jobs.py | 10 ++++------ .../aiplatform/model_monitoring/objective.py | 16 ++++++++-------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/google/cloud/aiplatform/__init__.py b/google/cloud/aiplatform/__init__.py index 4ca60ff30a..fadf469134 100644 --- a/google/cloud/aiplatform/__init__.py +++ b/google/cloud/aiplatform/__init__.py @@ -116,6 +116,7 @@ "HyperparameterTuningJob", "Model", "ModelEvaluation", + "ModelDeploymentMonitoringJob", "PipelineJob", "TabularDataset", "Tensorboard", diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 7dfbf61965..21dda7d894 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -1951,7 +1951,7 @@ def __init__( location=location, credentials=credentials, ) - self._gca_resource = self._get_gca_resource(resource_name=job_name) + self._gca_resource = self._get_gca_resource(resource_name=self.job_name) @classmethod def _parse_configs( @@ -2055,7 +2055,7 @@ def _empty_constructor( def create( cls, display_name: str, - endpoint: Union[str, "models.Endpoint"], + endpoint: Union[str, "aiplatform.Endpoint"], objective_configs: Union[ model_monitoring.EndpointObjectiveConfig, Dict[str, model_monitoring.EndpointObjectiveConfig], @@ -2227,9 +2227,7 @@ def create( _LOGGER.log_create_with_lro(cls) if int(endpoint): - formatted_endpoint = ( - f"projects/{project}/locations/{location}/endpoints/{endpoint}" - ) + endpoint = f"projects/{project}/locations/{location}/endpoints/{endpoint}" gapic_mdm_job = ( gca_model_deployment_monitoring_job.ModelDeploymentMonitoringJob( @@ -2296,7 +2294,7 @@ def update( ) -> "ModelDeploymentMonitoringJob": """""" current_job = self.api_client.get_model_deployment_monitoring_job( - name=slef.model_deployment_monitoring_job_name + name=self.model_deployment_monitoring_job_name ) update_mask: List[str] = [] if display_name: diff --git a/google/cloud/aiplatform/model_monitoring/objective.py b/google/cloud/aiplatform/model_monitoring/objective.py index 3a51da3cd7..c653a49e58 100644 --- a/google/cloud/aiplatform/model_monitoring/objective.py +++ b/google/cloud/aiplatform/model_monitoring/objective.py @@ -107,11 +107,13 @@ def as_proto(self): class _ObjectiveConfig(abc.ABC): def __init__( self, - skew_detection_config: Optional["model_monitoring._SkewDetectionConfig"] = None, + skew_detection_config: Optional[ + "gca_model_monitoring._SkewDetectionConfig" + ] = None, drift_detection_config: Optional[ - "model_monitoring._DriftDetectionConfig" + "gca_model_monitoring._DriftDetectionConfig" ] = None, - explanation_config: Optional["model_monitoring._ExplanationConfig"] = None, + explanation_config: Optional["gca_model_monitoring._ExplanationConfig"] = None, ): self.skew_detection_config = skew_detection_config self.drift_detection_config = drift_detection_config @@ -119,7 +121,6 @@ def __init__( def as_proto(self): training_dataset = None - # print(self.skew_detection_config.target_field) if self.skew_detection_config is not None: training_dataset = ( gca_model_monitoring.ModelMonitoringObjectiveConfig.TrainingDataset( @@ -136,7 +137,6 @@ def as_proto(self): ) else: training_dataset.dataset = self.skew_detection_config.data_source - # print(training_dataset) return gca_model_monitoring.ModelMonitoringObjectiveConfig( training_dataset=training_dataset, training_prediction_skew_detection_config=self.skew_detection_config.as_proto(), @@ -253,13 +253,13 @@ class EndpointObjectiveConfig(_ObjectiveConfig): def __init__( self, skew_detection_config: Optional[ - "model_monitoring.EndpointSkewDetectionConfig" + "gca_model_monitoring.EndpointSkewDetectionConfig" ] = None, drift_detection_config: Optional[ - "model_monitoring.EndpointDriftDetectionConfig" + "gca_model_monitoring.EndpointDriftDetectionConfig" ] = None, explanation_config: Optional[ - "model_monitoring.EndpointExplanationConfig" + "gca_model_monitoring.EndpointExplanationConfig" ] = None, ): """""" From 982d7fcade7d5876b594bbe7fc7d793949c9bca2 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Mon, 13 Jun 2022 23:28:21 +0000 Subject: [PATCH 12/53] added endpoint path resolution logic --- google/cloud/aiplatform/jobs.py | 50 ++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 21dda7d894..cebe63ed18 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -21,6 +21,7 @@ import copy import datetime import time +import re from google.cloud import storage from google.cloud import bigquery @@ -1952,6 +1953,20 @@ def __init__( credentials=credentials, ) self._gca_resource = self._get_gca_resource(resource_name=self.job_name) + self._endpoint_resource_name = "" + + @classmethod + def _get_endpoint_resource_name(cls, endpoint: Union[str, "aiplatform.Endpoint"]): + endpoint_resource_string = None + if isinstance(endpoint, str): + if re.match(r"[0-9]", endpoint): + parent = aiplatform.initializer.global_config.common_location_path() + endpoint_resource_string = f"{parent}/endpoints/{endpoint}" + elif re.match(r"projects/*/locations/*/endpoints/*", endpoint): + endpoint_resource_string = endpoint + elif isinstance(endpoint, aiplatform.Endpoint): + endpoint_resource_string = endpoint.resource_name + return endpoint_resource_string @classmethod def _parse_configs( @@ -1960,27 +1975,26 @@ def _parse_configs( model_monitoring.EndpointObjectiveConfig, Dict[str, model_monitoring.EndpointObjectiveConfig], ], - project, - location, - endpoint, - deployed_model_ids, + endpoint: Union[str, "aiplatform.Endpoint"], + deployed_model_ids: Optional[List[str]] = None, ): all_configs = [] all_models = [] default_endpoint = "aiplatform.googleapis.com" - client_options = dict(api_endpoint=f"{location}-{default_endpoint}") + client_options = dict( + api_endpoint=f"{aiplatform.initializer.global_config._location}-{default_endpoint}" + ) client = endpoint_service_client.EndpointServiceClient( client_options=client_options ) - parent = f"projects/{project}/locations/{location}" - response = client.get_endpoint(name=f"{parent}/endpoints/{endpoint}") + response = client.get_endpoint(name=cls._get_endpoint_resource_name(endpoint)) for model in response.deployed_models: all_models.append(model.id) ## when same objective config is applied to ALL models if ( isinstance(objective_configs, model_monitoring.EndpointObjectiveConfig) - and "*" in deployed_model_ids + and deployed_model_ids is None ): for model in all_models: all_configs.append( @@ -1993,7 +2007,7 @@ def _parse_configs( ## when same objective config is applied to SOME models elif ( isinstance(objective_configs, model_monitoring.EndpointObjectiveConfig) - and "*" not in deployed_model_ids + and deployed_model_ids is not None ): for model in deployed_model_ids: assert model in all_models @@ -2015,7 +2029,6 @@ def _parse_configs( ) ) - print(all_configs) return all_configs @classmethod @@ -2064,7 +2077,7 @@ def create( monitor_interval: int, schedule_config: model_monitoring.ScheduleConfig, timeout: float = None, - deployed_model_ids: Optional[List[str]] = ["*"], + deployed_model_ids: Optional[List[str]] = None, metadata: Optional[Sequence[Tuple[str, str]]] = None, alert_config: Optional[model_monitoring.EmailAlertConfig] = None, predict_instance_schema_uri: Optional[str] = None, @@ -2089,7 +2102,7 @@ def create( UTF-8 characters. Display name of a ModelDeploymentMonitoringJob. - endpoint (Union[str, "models.Endpoint"]): + endpoint (Union[str, "aiplatform.Endpoint"]): Endpoint resource name. Format: ``projects/{project}/locations/{location}/endpoints/{endpoint}`` @@ -2208,7 +2221,7 @@ def create( ) mdm_objective_config_seq = cls._parse_configs( - objective_configs, project, location, endpoint, deployed_model_ids + objective_configs, endpoint, deployed_model_ids ) empty_mdm_job = cls._empty_constructor( @@ -2226,13 +2239,12 @@ def create( _LOGGER.log_create_with_lro(cls) - if int(endpoint): - endpoint = f"projects/{project}/locations/{location}/endpoints/{endpoint}" + endpoint_resource_name = cls._get_endpoint_resource_name(endpoint) gapic_mdm_job = ( gca_model_deployment_monitoring_job.ModelDeploymentMonitoringJob( display_name=display_name, - endpoint=endpoint, + endpoint=endpoint_resource_name, model_deployment_monitoring_objective_configs=mdm_objective_config_seq, logging_sampling_strategy=logging_sampling_strategy.as_proto(), model_deployment_monitoring_schedule_config=schedule_config.as_proto(), @@ -2264,6 +2276,8 @@ def create( "View Model Deployment Monitoring Job:\n%s" % mdm_job._dashboard_uri() ) + mdm_job._endpoint_resource_name = endpoint_resource_name + return mdm_job def update( @@ -2323,7 +2337,9 @@ def update( if objective_configs: update_mask.append("model_deployment_monitoring_objective_configs") current_job.model_deployment_monitoring_objective_configs = ( - self._parse_configs(objective_configs) + ModelDeploymentMonitoringJob._parse_configs( + objective_configs, self._endpoint_resource_name + ) ) self.api_client.update_model_deployment_monitoring_job( model_deployment_monitoring_job=current_job, update_mask=update_mask From 483e6c69ab1e7ea88e9b47f56206d73cc69a836f Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Tue, 14 Jun 2022 00:20:23 +0000 Subject: [PATCH 13/53] excluding uninitialized optional arguments for as_proto methods in objective config --- .../aiplatform/model_monitoring/objective.py | 42 +++++++++++-------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/google/cloud/aiplatform/model_monitoring/objective.py b/google/cloud/aiplatform/model_monitoring/objective.py index c653a49e58..caf2e2e808 100644 --- a/google/cloud/aiplatform/model_monitoring/objective.py +++ b/google/cloud/aiplatform/model_monitoring/objective.py @@ -53,13 +53,14 @@ def as_proto(self): for key in self.skew_thresholds.keys(): skew_threshold = gca_threshold_config(value=self.skew_thresholds[key]) skew_thresholds_mapping[key] = skew_threshold - for key in self.attribute_skew_thresholds.keys(): - attribution_score_skew_threshold = gca_threshold_config( - value=self.attribute_skew_thresholds[key] - ) - attribution_score_skew_thresholds_mapping[ - key - ] = attribution_score_skew_threshold + if self.attribute_skew_thresholds is not None: + for key in self.attribute_skew_thresholds.keys(): + attribution_score_skew_threshold = gca_threshold_config( + value=self.attribute_skew_thresholds[key] + ) + attribution_score_skew_thresholds_mapping[ + key + ] = attribution_score_skew_threshold return gca_model_monitoring.ModelMonitoringObjectiveConfig.TrainingPredictionSkewDetectionConfig( skew_thresholds=skew_thresholds_mapping, attribution_score_skew_thresholds=attribution_score_skew_thresholds_mapping, @@ -81,13 +82,14 @@ def as_proto(self): for key in self.drift_thresholds.keys(): drift_threshold = gca_threshold_config(value=self.drift_thresholds[key]) drift_thresholds_mapping[key] = drift_threshold - for key in self.attribute_drift_thresholds.keys(): - attribution_score_drift_threshold = gca_threshold_config( - value=self.attribute_drift_thresholds[key] - ) - attribution_score_drift_thresholds_mapping[ - key - ] = attribution_score_drift_threshold + if self.attribute_drift_thresholds is not None: + for key in self.attribute_drift_thresholds.keys(): + attribution_score_drift_threshold = gca_threshold_config( + value=self.attribute_drift_thresholds[key] + ) + attribution_score_drift_thresholds_mapping[ + key + ] = attribution_score_drift_threshold return gca_model_monitoring.ModelMonitoringObjectiveConfig.PredictionDriftDetectionConfig( drift_thresholds=drift_thresholds_mapping, attribution_score_drift_thresholds=attribution_score_drift_thresholds_mapping, @@ -139,9 +141,15 @@ def as_proto(self): training_dataset.dataset = self.skew_detection_config.data_source return gca_model_monitoring.ModelMonitoringObjectiveConfig( training_dataset=training_dataset, - training_prediction_skew_detection_config=self.skew_detection_config.as_proto(), - prediction_drift_detection_config=self.drift_detection_config.as_proto(), - explanation_config=self.explanation_config.as_proto(), + training_prediction_skew_detection_config=self.skew_detection_config.as_proto() + if self.detection_config is not None + else None, + prediction_drift_detection_config=self.drift_detection_config.as_proto() + if self.drift_detection_config is not None + else None, + explanation_config=self.explanation_config.as_proto() + if self.explanation_config is not None + else None, ) From 8f362133e85f09a808ea8168865503cd5981a450 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Tue, 14 Jun 2022 00:42:55 +0000 Subject: [PATCH 14/53] fixing typo in class variable --- google/cloud/aiplatform/model_monitoring/objective.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/aiplatform/model_monitoring/objective.py b/google/cloud/aiplatform/model_monitoring/objective.py index caf2e2e808..b6316e6cf9 100644 --- a/google/cloud/aiplatform/model_monitoring/objective.py +++ b/google/cloud/aiplatform/model_monitoring/objective.py @@ -142,7 +142,7 @@ def as_proto(self): return gca_model_monitoring.ModelMonitoringObjectiveConfig( training_dataset=training_dataset, training_prediction_skew_detection_config=self.skew_detection_config.as_proto() - if self.detection_config is not None + if self.skew_detection_config is not None else None, prediction_drift_detection_config=self.drift_detection_config.as_proto() if self.drift_detection_config is not None From 79b8b7883fcc8ddbcdb6749a21fb39b21b0bba3f Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Wed, 15 Jun 2022 01:34:13 +0000 Subject: [PATCH 15/53] adding more upstream error handling --- google/cloud/aiplatform/jobs.py | 4 + .../aiplatform/model_monitoring/alert.py | 8 +- .../aiplatform/model_monitoring/objective.py | 113 ++++++++++-------- .../aiplatform/model_monitoring/sampling.py | 2 +- 4 files changed, 72 insertions(+), 55 deletions(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index cebe63ed18..9b275e5901 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -1966,6 +1966,10 @@ def _get_endpoint_resource_name(cls, endpoint: Union[str, "aiplatform.Endpoint"] endpoint_resource_string = endpoint elif isinstance(endpoint, aiplatform.Endpoint): endpoint_resource_string = endpoint.resource_name + else: + raise ValueError( + "Invalid value for endpoint. `endpoint` needs to be one of numeric ID, well-formatted path, or an instance of aiplatform.Endpoint. If providing the full path, it needs to follow the format projects/$PROJECT/locations/$LOCATION/endpoints/$ENDPOINT_ID" + ) return endpoint_resource_string @classmethod diff --git a/google/cloud/aiplatform/model_monitoring/alert.py b/google/cloud/aiplatform/model_monitoring/alert.py index d7390fafa5..f05e87edc7 100644 --- a/google/cloud/aiplatform/model_monitoring/alert.py +++ b/google/cloud/aiplatform/model_monitoring/alert.py @@ -25,19 +25,21 @@ class _AlertConfig(abc.ABC): """An abstract class for setting model monitoring alert config""" - def __init__(self, enable_logging: Optional[bool] = None): + def __init__(self, enable_logging: Optional[bool] = False): self.enable_logging = enable_logging class EmailAlertConfig(_AlertConfig): - def __init__(self, user_emails: List[str], enable_logging: Optional[bool] = None): + def __init__( + self, user_emails: List[str] = [], enable_logging: Optional[bool] = False + ): """Initializer for EmailAlertConfig Args: user_emails (List[str]): The email addresses to send the alert to. enable_logging (bool): - Optional. Streams detected anomalies to Cloud Logging. The anomalies will be + Optional. Defaults to False. Streams detected anomalies to Cloud Logging. The anomalies will be put into json payload encoded from proto [google.cloud.aiplatform.logging.ModelMonitoringAnomaliesLogEntry][]. This can be further sync'd to Pub/Sub or any other services diff --git a/google/cloud/aiplatform/model_monitoring/objective.py b/google/cloud/aiplatform/model_monitoring/objective.py index b6316e6cf9..2df17ab236 100644 --- a/google/cloud/aiplatform/model_monitoring/objective.py +++ b/google/cloud/aiplatform/model_monitoring/objective.py @@ -18,11 +18,6 @@ import abc from typing import Optional, Dict -# from google.cloud.aiplatform.compat.types import ( -# model_monitoring as gca_model_monitoring, -# ) -# from google.cloud.aiplatform_v1.types import ThresholdConfig as gca_threshold_config -# from google.cloud.aiplatform_v1.types.io import BigQuerySource from google.cloud.aiplatform_v1.types import ( io as gca_io, ThresholdConfig as gca_threshold_config, @@ -35,17 +30,17 @@ def __init__( self, data_source: str, skew_thresholds: Dict[str, float], - attribute_skew_thresholds: Optional[Dict[str, float]] = None, + target_field: str, + attribute_skew_thresholds: Dict[str, float], data_format: Optional[str] = None, - target_field: Optional[str] = None, ): - """""" - # print(skew_thresholds) + """Base class for training-serving skew detection""" self.data_source = data_source self.skew_thresholds = skew_thresholds self.attribute_skew_thresholds = attribute_skew_thresholds self.data_format = data_format self.target_field = target_field + self.training_dataset = None def as_proto(self): skew_thresholds_mapping = {} @@ -53,14 +48,13 @@ def as_proto(self): for key in self.skew_thresholds.keys(): skew_threshold = gca_threshold_config(value=self.skew_thresholds[key]) skew_thresholds_mapping[key] = skew_threshold - if self.attribute_skew_thresholds is not None: - for key in self.attribute_skew_thresholds.keys(): - attribution_score_skew_threshold = gca_threshold_config( - value=self.attribute_skew_thresholds[key] - ) - attribution_score_skew_thresholds_mapping[ - key - ] = attribution_score_skew_threshold + for key in self.attribute_skew_thresholds.keys(): + attribution_score_skew_threshold = gca_threshold_config( + value=self.attribute_skew_thresholds[key] + ) + attribution_score_skew_thresholds_mapping[ + key + ] = attribution_score_skew_threshold return gca_model_monitoring.ModelMonitoringObjectiveConfig.TrainingPredictionSkewDetectionConfig( skew_thresholds=skew_thresholds_mapping, attribution_score_skew_thresholds=attribution_score_skew_thresholds_mapping, @@ -71,7 +65,7 @@ class _DriftDetectionConfig(abc.ABC): def __init__( self, drift_thresholds: Dict[str, float], - attribute_drift_thresholds: Optional[Dict[str, float]] = None, + attribute_drift_thresholds: Dict[str, float], ): self.drift_thresholds = drift_thresholds self.attribute_drift_thresholds = attribute_drift_thresholds @@ -82,14 +76,13 @@ def as_proto(self): for key in self.drift_thresholds.keys(): drift_threshold = gca_threshold_config(value=self.drift_thresholds[key]) drift_thresholds_mapping[key] = drift_threshold - if self.attribute_drift_thresholds is not None: - for key in self.attribute_drift_thresholds.keys(): - attribution_score_drift_threshold = gca_threshold_config( - value=self.attribute_drift_thresholds[key] - ) - attribution_score_drift_thresholds_mapping[ - key - ] = attribution_score_drift_threshold + for key in self.attribute_drift_thresholds.keys(): + attribution_score_drift_threshold = gca_threshold_config( + value=self.attribute_drift_thresholds[key] + ) + attribution_score_drift_thresholds_mapping[ + key + ] = attribution_score_drift_threshold return gca_model_monitoring.ModelMonitoringObjectiveConfig.PredictionDriftDetectionConfig( drift_thresholds=drift_thresholds_mapping, attribution_score_drift_thresholds=attribution_score_drift_thresholds_mapping, @@ -124,21 +117,7 @@ def __init__( def as_proto(self): training_dataset = None if self.skew_detection_config is not None: - training_dataset = ( - gca_model_monitoring.ModelMonitoringObjectiveConfig.TrainingDataset( - target_field=self.skew_detection_config.target_field - ) - ) - if "bq:/" in self.skew_detection_config.data_source: - training_dataset.bigquery_source = gca_io.BigQuerySource( - input_uri=self.skew_detection_config.data_source - ) - elif "gs:/" in self.skew_detection_config.data_source: - training_dataset.gcs_source = gca_io.GcsSource( - uris=[self.skew_detection_config.data_source] - ) - else: - training_dataset.dataset = self.skew_detection_config.data_source + training_dataset = self.training_dataset return gca_model_monitoring.ModelMonitoringObjectiveConfig( training_dataset=training_dataset, training_prediction_skew_detection_config=self.skew_detection_config.as_proto() @@ -164,10 +143,10 @@ class EndpointSkewDetectionConfig(_SkewDetectionConfig): def __init__( self, data_source: str, - skew_thresholds: Optional[Dict[str, float]] = None, - attribute_skew_thresholds: Optional[Dict[str, float]] = None, + target_field: str, + skew_thresholds: Optional[Dict[str, float]] = {}, + attribute_skew_thresholds: Optional[Dict[str, float]] = {}, data_format: Optional[str] = None, - target_field: Optional[str] = None, ): """Initializer for EndpointSkewDetectionConfig @@ -175,6 +154,11 @@ def __init__( data_source (str): Path to training dataset. + target_field (str): + The target field name the model is to + predict. This field will be excluded when doing + Predict and (or) Explain for the training data. + skew_thresholds (Dict[str, float]): Optional. Key is the feature name and value is the threshold. If a feature needs to be monitored @@ -200,13 +184,14 @@ def __init__( "csv" The source file is a CSV file. - target_field (str): - The target field name the model is to - predict. This field will be excluded when doing - Predict and (or) Explain for the training data. + "jsonl" + The source file is a JSONL file. Returns: An instance of EndpointSkewDetectionConfig + + Raises: + ValueError """ super().__init__( data_source, @@ -216,6 +201,32 @@ def __init__( target_field, ) + training_dataset = ( + gca_model_monitoring.ModelMonitoringObjectiveConfig.TrainingDataset( + target_field=self.skew_detection_config.target_field + ) + ) + if data_source.startswith("bq:/"): + training_dataset.bigquery_source = gca_io.BigQuerySource( + input_uri=data_source + ) + elif data_source.startswith("gs:/"): + training_dataset.gcs_source = gca_io.GcsSource( + uris=[self.skew_detection_config.data_source] + ) + if data_format is not None and data_format not in [ + "tf-record", + "csv", + "jsonl", + ]: + raise ValueError( + "Unsupported value. `data_format` must be one of 'tf-record', 'csv', or 'jsonl'" + ) + training_dataset.data_format = data_format + else: + training_dataset.dataset = data_source + self.training_dataset = training_dataset + class EndpointDriftDetectionConfig(_DriftDetectionConfig): """A class that configures prediction drift detection for models deployed to an endpoint. @@ -228,8 +239,8 @@ class EndpointDriftDetectionConfig(_DriftDetectionConfig): def __init__( self, - drift_thresholds: Optional[Dict[str, float]] = None, - attribute_drift_thresholds: Optional[Dict[str, float]] = None, + drift_thresholds: Optional[Dict[str, float]] = {}, + attribute_drift_thresholds: Optional[Dict[str, float]] = {}, ): """Initializer for EndpointDriftDetectionConfig @@ -256,7 +267,7 @@ def __init__(self): class EndpointObjectiveConfig(_ObjectiveConfig): - """A class that captures skew detection, drift detection, and explaination configs.""" + """A class that captures skew detection, drift detection, and explanation configs.""" def __init__( self, diff --git a/google/cloud/aiplatform/model_monitoring/sampling.py b/google/cloud/aiplatform/model_monitoring/sampling.py index 89513c6ee7..c3cd5f47b0 100644 --- a/google/cloud/aiplatform/model_monitoring/sampling.py +++ b/google/cloud/aiplatform/model_monitoring/sampling.py @@ -27,7 +27,7 @@ class _SamplingStrategy(abc.ABC): class RandomSampleConfig(_SamplingStrategy): - def __init__(self, sample_rate: Optional[float] = None): + def __init__(self, sample_rate: Optional[float] = 1): """Initializer for RandomSampleConfig Args: From a5d2b08e6a42f656ecb09ddf2785632b3f1c3abd Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Wed, 15 Jun 2022 20:23:27 +0000 Subject: [PATCH 16/53] fixing errors with runtime type checks --- google/cloud/aiplatform/jobs.py | 7 ++----- google/cloud/aiplatform/model_monitoring/objective.py | 6 +++--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 9b275e5901..6023583918 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -1957,12 +1957,12 @@ def __init__( @classmethod def _get_endpoint_resource_name(cls, endpoint: Union[str, "aiplatform.Endpoint"]): - endpoint_resource_string = None + endpoint_resource_string = "" if isinstance(endpoint, str): if re.match(r"[0-9]", endpoint): parent = aiplatform.initializer.global_config.common_location_path() endpoint_resource_string = f"{parent}/endpoints/{endpoint}" - elif re.match(r"projects/*/locations/*/endpoints/*", endpoint): + elif re.match(r"projects/.+?/locations/.+?/endpoints/.+?", endpoint): endpoint_resource_string = endpoint elif isinstance(endpoint, aiplatform.Endpoint): endpoint_resource_string = endpoint.resource_name @@ -2244,7 +2244,6 @@ def create( _LOGGER.log_create_with_lro(cls) endpoint_resource_name = cls._get_endpoint_resource_name(endpoint) - gapic_mdm_job = ( gca_model_deployment_monitoring_job.ModelDeploymentMonitoringJob( display_name=display_name, @@ -2286,8 +2285,6 @@ def create( def update( self, - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), display_name: Optional[str] = None, schedule_config: Optional[model_monitoring.ScheduleConfig] = None, alert_config: Optional[model_monitoring.EmailAlertConfig] = None, diff --git a/google/cloud/aiplatform/model_monitoring/objective.py b/google/cloud/aiplatform/model_monitoring/objective.py index 2df17ab236..a131ce96f5 100644 --- a/google/cloud/aiplatform/model_monitoring/objective.py +++ b/google/cloud/aiplatform/model_monitoring/objective.py @@ -117,7 +117,7 @@ def __init__( def as_proto(self): training_dataset = None if self.skew_detection_config is not None: - training_dataset = self.training_dataset + training_dataset = self.skew_detection_config.training_dataset return gca_model_monitoring.ModelMonitoringObjectiveConfig( training_dataset=training_dataset, training_prediction_skew_detection_config=self.skew_detection_config.as_proto() @@ -196,14 +196,14 @@ def __init__( super().__init__( data_source, skew_thresholds, + target_field, attribute_skew_thresholds, data_format, - target_field, ) training_dataset = ( gca_model_monitoring.ModelMonitoringObjectiveConfig.TrainingDataset( - target_field=self.skew_detection_config.target_field + target_field=target_field ) ) if data_source.startswith("bq:/"): From 99987d3f2ad70f462296eadd3850de88fd44e20e Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Thu, 16 Jun 2022 01:32:30 +0000 Subject: [PATCH 17/53] fixed runtime errors in update and pause functions --- google/cloud/aiplatform/jobs.py | 42 ++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 6023583918..da2cabb9b0 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -28,6 +28,7 @@ from google.auth import credentials as auth_credentials from google.protobuf import duration_pb2 # type: ignore +from google.protobuf import field_mask_pb2 # type: ignore from google.rpc import status_pb2 from google.cloud import aiplatform @@ -1952,7 +1953,9 @@ def __init__( location=location, credentials=credentials, ) - self._gca_resource = self._get_gca_resource(resource_name=self.job_name) + self._gca_resource = self._get_gca_resource( + resource_name=model_deployment_monitoring_job_name + ) self._endpoint_resource_name = "" @classmethod @@ -1985,6 +1988,10 @@ def _parse_configs( all_configs = [] all_models = [] default_endpoint = "aiplatform.googleapis.com" + if aiplatform.initializer.global_config._location is None: + raise ValueError( + "Error parsing model monitoring objective configs: project location is not set" + ) client_options = dict( api_endpoint=f"{aiplatform.initializer.global_config._location}-{default_endpoint}" ) @@ -2309,7 +2316,7 @@ def update( ) -> "ModelDeploymentMonitoringJob": """""" current_job = self.api_client.get_model_deployment_monitoring_job( - name=self.model_deployment_monitoring_job_name + name=self._gca_resource.name ) update_mask: List[str] = [] if display_name: @@ -2339,27 +2346,40 @@ def update( update_mask.append("model_deployment_monitoring_objective_configs") current_job.model_deployment_monitoring_objective_configs = ( ModelDeploymentMonitoringJob._parse_configs( - objective_configs, self._endpoint_resource_name + objective_configs, current_job.endpoint, deployed_model_ids ) ) self.api_client.update_model_deployment_monitoring_job( - model_deployment_monitoring_job=current_job, update_mask=update_mask + model_deployment_monitoring_job=current_job, + update_mask=field_mask_pb2.FieldMask(paths=update_mask), ) def pause(self) -> "ModelDeploymentMonitoringJob": """""" - self.api_client.pause_model_deployment_monitoring_job( - self.model_deployment_monitoring_job_name - ) + if self.state == gca_job_state.JobState.JOB_STATE_RUNNING: + self.api_client.pause_model_deployment_monitoring_job( + name=self._gca_resource.name + ) + else: + raise RuntimeError( + "The monitoring job can only be paused under running / pending state, the current state is: %s" + % self.state + ) def resume(self) -> "ModelDeploymentMonitoringJob": """""" - self.api_client.resume_model_deployment_monitoring_job( - self.model_deployment_monitoring_job_name - ) + if self.state == gca_job_state.JobState.JOB_STATE_PAUSED: + self.api_client.resume_model_deployment_monitoring_job( + name=self._gca_resource.name + ) + else: + raise RuntimeError( + "The monitoring job can only be resumed under paused state" + ) def delete(self) -> "ModelDeploymentMonitoringJob": """""" + self.pause() self.api_client.delete_model_deployment_monitoring_job( - self.model_deployment_monitoring_job_name + name=self._gca_resource.name ) From 046af5ad12b34cc88690491b4a90153ecd239e20 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Thu, 16 Jun 2022 22:38:29 +0000 Subject: [PATCH 18/53] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .kokoro/samples/python3.6/common.cfg | 40 +++++++++++++++++++ .kokoro/samples/python3.6/continuous.cfg | 7 ++++ .kokoro/samples/python3.6/periodic-head.cfg | 11 +++++ .kokoro/samples/python3.6/presubmit.cfg | 6 +++ samples/model-builder/noxfile.py | 2 +- samples/snippets/noxfile.py | 2 +- .../templates/install_deps.tmpl.rst | 2 +- 7 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 .kokoro/samples/python3.6/common.cfg create mode 100644 .kokoro/samples/python3.6/continuous.cfg create mode 100644 .kokoro/samples/python3.6/periodic-head.cfg create mode 100644 .kokoro/samples/python3.6/presubmit.cfg diff --git a/.kokoro/samples/python3.6/common.cfg b/.kokoro/samples/python3.6/common.cfg new file mode 100644 index 0000000000..72bfadc9f4 --- /dev/null +++ b/.kokoro/samples/python3.6/common.cfg @@ -0,0 +1,40 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.6" +} + +# Declare build specific Cloud project. +env_vars: { + key: "BUILD_SPECIFIC_GCLOUD_PROJECT" + value: "ucaip-sample-tests" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-aiplatform/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-aiplatform/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.6/continuous.cfg b/.kokoro/samples/python3.6/continuous.cfg new file mode 100644 index 0000000000..7218af1499 --- /dev/null +++ b/.kokoro/samples/python3.6/continuous.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + diff --git a/.kokoro/samples/python3.6/periodic-head.cfg b/.kokoro/samples/python3.6/periodic-head.cfg new file mode 100644 index 0000000000..88d5235e34 --- /dev/null +++ b/.kokoro/samples/python3.6/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-aiplatform/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/samples/python3.6/presubmit.cfg b/.kokoro/samples/python3.6/presubmit.cfg new file mode 100644 index 0000000000..a1c8d9759c --- /dev/null +++ b/.kokoro/samples/python3.6/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/samples/model-builder/noxfile.py b/samples/model-builder/noxfile.py index 5fcb9d7461..38bb0a572b 100644 --- a/samples/model-builder/noxfile.py +++ b/samples/model-builder/noxfile.py @@ -89,7 +89,7 @@ def get_pytest_env_vars() -> Dict[str, str]: # DO NOT EDIT - automatically generated. # All versions used to test samples. -ALL_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] +ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] # Any default versions that should be ignored. IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py index 5fcb9d7461..38bb0a572b 100644 --- a/samples/snippets/noxfile.py +++ b/samples/snippets/noxfile.py @@ -89,7 +89,7 @@ def get_pytest_env_vars() -> Dict[str, str]: # DO NOT EDIT - automatically generated. # All versions used to test samples. -ALL_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] +ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] # Any default versions that should be ignored. IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] diff --git a/scripts/readme-gen/templates/install_deps.tmpl.rst b/scripts/readme-gen/templates/install_deps.tmpl.rst index 6f069c6c87..275d649890 100644 --- a/scripts/readme-gen/templates/install_deps.tmpl.rst +++ b/scripts/readme-gen/templates/install_deps.tmpl.rst @@ -12,7 +12,7 @@ Install Dependencies .. _Python Development Environment Setup Guide: https://cloud.google.com/python/setup -#. Create a virtualenv. Samples are compatible with Python 3.7+. +#. Create a virtualenv. Samples are compatible with Python 3.6+. .. code-block:: bash From 8e85f5e44f0b91ef28480118037b8d88971bb51d Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Thu, 16 Jun 2022 22:38:33 +0000 Subject: [PATCH 19/53] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .kokoro/samples/python3.6/common.cfg | 40 +++++++++++++++++++ .kokoro/samples/python3.6/continuous.cfg | 7 ++++ .kokoro/samples/python3.6/periodic-head.cfg | 11 +++++ .kokoro/samples/python3.6/presubmit.cfg | 6 +++ samples/model-builder/noxfile.py | 2 +- samples/snippets/noxfile.py | 2 +- .../templates/install_deps.tmpl.rst | 2 +- 7 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 .kokoro/samples/python3.6/common.cfg create mode 100644 .kokoro/samples/python3.6/continuous.cfg create mode 100644 .kokoro/samples/python3.6/periodic-head.cfg create mode 100644 .kokoro/samples/python3.6/presubmit.cfg diff --git a/.kokoro/samples/python3.6/common.cfg b/.kokoro/samples/python3.6/common.cfg new file mode 100644 index 0000000000..72bfadc9f4 --- /dev/null +++ b/.kokoro/samples/python3.6/common.cfg @@ -0,0 +1,40 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.6" +} + +# Declare build specific Cloud project. +env_vars: { + key: "BUILD_SPECIFIC_GCLOUD_PROJECT" + value: "ucaip-sample-tests" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-aiplatform/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-aiplatform/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.6/continuous.cfg b/.kokoro/samples/python3.6/continuous.cfg new file mode 100644 index 0000000000..7218af1499 --- /dev/null +++ b/.kokoro/samples/python3.6/continuous.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + diff --git a/.kokoro/samples/python3.6/periodic-head.cfg b/.kokoro/samples/python3.6/periodic-head.cfg new file mode 100644 index 0000000000..88d5235e34 --- /dev/null +++ b/.kokoro/samples/python3.6/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-aiplatform/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/samples/python3.6/presubmit.cfg b/.kokoro/samples/python3.6/presubmit.cfg new file mode 100644 index 0000000000..a1c8d9759c --- /dev/null +++ b/.kokoro/samples/python3.6/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/samples/model-builder/noxfile.py b/samples/model-builder/noxfile.py index 5fcb9d7461..38bb0a572b 100644 --- a/samples/model-builder/noxfile.py +++ b/samples/model-builder/noxfile.py @@ -89,7 +89,7 @@ def get_pytest_env_vars() -> Dict[str, str]: # DO NOT EDIT - automatically generated. # All versions used to test samples. -ALL_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] +ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] # Any default versions that should be ignored. IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py index 5fcb9d7461..38bb0a572b 100644 --- a/samples/snippets/noxfile.py +++ b/samples/snippets/noxfile.py @@ -89,7 +89,7 @@ def get_pytest_env_vars() -> Dict[str, str]: # DO NOT EDIT - automatically generated. # All versions used to test samples. -ALL_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] +ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] # Any default versions that should be ignored. IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] diff --git a/scripts/readme-gen/templates/install_deps.tmpl.rst b/scripts/readme-gen/templates/install_deps.tmpl.rst index 6f069c6c87..275d649890 100644 --- a/scripts/readme-gen/templates/install_deps.tmpl.rst +++ b/scripts/readme-gen/templates/install_deps.tmpl.rst @@ -12,7 +12,7 @@ Install Dependencies .. _Python Development Environment Setup Guide: https://cloud.google.com/python/setup -#. Create a virtualenv. Samples are compatible with Python 3.7+. +#. Create a virtualenv. Samples are compatible with Python 3.6+. .. code-block:: bash From debddd64c1dd60119c85beaaf941f8ed7da8c0e9 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Thu, 16 Jun 2022 22:43:18 +0000 Subject: [PATCH 20/53] removing unused methods --- google/cloud/aiplatform/jobs.py | 101 +++++++++++++++----------------- 1 file changed, 46 insertions(+), 55 deletions(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 024c74d354..9d60ac6d2b 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -15,7 +15,7 @@ # limitations under the License. # -from typing import Iterable, Optional, Union, Sequence, Dict, List, Tuple +from typing import Iterable, Optional, Union, Sequence, Dict, List import abc import copy @@ -1958,6 +1958,7 @@ def __init__( @classmethod def _get_endpoint_resource_name(cls, endpoint: Union[str, "aiplatform.Endpoint"]): + """Helper function for validating endpoint resource name""" endpoint_resource_string = "" if isinstance(endpoint, str): if re.match(r"[0-9]", endpoint): @@ -1983,8 +1984,10 @@ def _parse_configs( endpoint: Union[str, "aiplatform.Endpoint"], deployed_model_ids: Optional[List[str]] = None, ): + """Helper function for matching objective configs with their corresponding models""" all_configs = [] all_models = [] + xai_enabled = [] default_endpoint = "aiplatform.googleapis.com" if aiplatform.initializer.global_config._location is None: raise ValueError( @@ -1999,6 +2002,8 @@ def _parse_configs( response = client.get_endpoint(name=cls._get_endpoint_resource_name(endpoint)) for model in response.deployed_models: all_models.append(model.id) + if model.explanation_spec.parameters == {}: + xai_enabled.append(model.id) ## when same objective config is applied to ALL models if ( @@ -2006,6 +2011,13 @@ def _parse_configs( and deployed_model_ids is None ): for model in all_models: + if ( + model not in xai_enabled + and objective_configs.explanation_config is not None + ): + raise RuntimeError( + "Invalid config for model. `explanation_config` should only be enabled if the model has `explanation_spec populated" + ) all_configs.append( gca_model_deployment_monitoring_job.ModelDeploymentMonitoringObjectiveConfig( deployed_model_id=model, @@ -2019,6 +2031,13 @@ def _parse_configs( and deployed_model_ids is not None ): for model in deployed_model_ids: + if ( + model not in xai_enabled + and objective_configs.explanation_config is not None + ): + raise RuntimeError( + "Invalid config for model. `explanation_config` should only be enabled if the model has `explanation_spec populated" + ) assert model in all_models all_configs.append( gca_model_deployment_monitoring_job.ModelDeploymentMonitoringObjectiveConfig( @@ -2031,6 +2050,13 @@ def _parse_configs( elif isinstance(objective_configs, Dict) and deployed_model_ids is None: assert all(model in all_models for model in objective_configs.keys()) for key in objective_configs.keys(): + if ( + model not in xai_enabled + and objective_configs[key].explanation_config is not None + ): + raise RuntimeError( + "Invalid config for model. `explanation_config` should only be enabled if the model has `explanation_spec populated" + ) all_configs.append( gca_model_deployment_monitoring_job.ModelDeploymentMonitoringObjectiveConfig( deployed_model_id=key, @@ -2040,43 +2066,9 @@ def _parse_configs( return all_configs - @classmethod - def _empty_constructor( - cls, - project: Optional[str] = None, - location: Optional[str] = None, - credentials: Optional[auth_credentials.Credentials] = None, - resource_name: Optional[str] = None, - ) -> "_Job": - """Initializes with all attributes set to None. - - The attributes should be populated after a future is complete. This allows - scheduling of additional API calls before the resource is created. - - Args: - project (str): Optional. Project of the resource noun. - location (str): Optional. The location of the resource noun. - credentials(google.auth.credentials.Credentials): - Optional. custom credentials to use when accessing interacting with - resource noun. - resource_name(str): Optional. A fully-qualified resource name or ID. - Returns: - An instance of this class with attributes set to None. - """ - self = super()._empty_constructor( - project=project, - location=location, - credentials=credentials, - resource_name=resource_name, - ) - - self._logged_web_access_uris = set() - return self - @classmethod def create( cls, - display_name: str, endpoint: Union[str, "aiplatform.Endpoint"], objective_configs: Union[ model_monitoring.EndpointObjectiveConfig, @@ -2086,8 +2078,8 @@ def create( monitor_interval: int, schedule_config: model_monitoring.ScheduleConfig, timeout: float = None, + display_name: Optional[str] = None, deployed_model_ids: Optional[List[str]] = None, - metadata: Optional[Sequence[Tuple[str, str]]] = None, alert_config: Optional[model_monitoring.EmailAlertConfig] = None, predict_instance_schema_uri: Optional[str] = None, sample_predict_instance: Optional[str] = None, @@ -2104,13 +2096,6 @@ def create( """Creates and launches a model monitoring job Args: - display_name (str): - The user-defined name of the - ModelDeploymentMonitoringJob. The name can be up - to 128 characters long and can be consist of any - UTF-8 characters. - Display name of a ModelDeploymentMonitoringJob. - endpoint (Union[str, "aiplatform.Endpoint"]): Endpoint resource name. Format: ``projects/{project}/locations/{location}/endpoints/{endpoint}`` @@ -2131,14 +2116,24 @@ def create( hour. This defines how often the monitoring jobs are triggered. + schedule_config (model_monitoring.schedule.ScheduleConfig): + Configures model monitoring job scheduling interval in hours. + This defines how often the monitoring jobs are triggered. + + timeout (float): timeout for the model monitoring job creation request + + display_name (str): + The user-defined name of the + ModelDeploymentMonitoringJob. The name can be up + to 128 characters long and can be consist of any + UTF-8 characters. + Display name of a ModelDeploymentMonitoringJob. + deployed_model_ids ([List[str]] = ["*"]): Optional. Use this argument to specify which deployed models to apply the objective config to. If left unspecified, the same config will be applied to all deployed models. - schedule_config (model_monitoring.schedule.ScheduleConfig): - Configures model monitoring job scheduling interval in hours. - This defines how often the monitoring jobs are triggered. alert_config (model_monitoring.alert.EmailAlertConfig): Optional. Configures how alerts are sent to the user. Right now only email alert is supported. @@ -2151,12 +2146,12 @@ def create( collected predict requests. sample_predict_instance (str): - Sample Predict instance, same format as PredictionRequest.instances, + Optional. Sample Predict instance, same format as PredictionRequest.instances, this can be set as a replacement of predict_instance_schema_uri If not set, the schema will be generated from collected predict requests. analysis_instance_schema_uri (str): - YAML schema file uri describing the format of a single + Optional. YAML schema file uri describing the format of a single instance that you want Tensorflow Data Validation (TFDV) to analyze. If this field is empty, all the feature data types are inferred from predict_instance_schema_uri, meaning that TFDV @@ -2167,7 +2162,7 @@ def create( fields in predict instance formatted as string. bigquery_tables_log_ttl (int): - The TTL(time to live) of BigQuery tables in user projects + Optional. The TTL(time to live) of BigQuery tables in user projects which stores logs. A day is the basic unit of the TTL and we take the ceil of TTL/86400(a day). e.g. { second: 3600} indicates ttl = 1 @@ -2201,15 +2196,11 @@ def create( ModelDeploymentMonitoringJob will be secured by this key. - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), - project: Optional[str] = None, - location: Optional[str] = None, - credentials: Optional[auth_credentials.Credentials] = None, - Returns: An instance of ModelDeploymentMonitoringJob + Raises: + ValueError from endpoint resource name and data source path validations """ if not display_name: display_name = cls._generate_display_name() From 1409729ad1eda3bc80eff6f2d42fffa25b971661 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Fri, 17 Jun 2022 01:13:17 -0700 Subject: [PATCH 21/53] fixed job delete method --- google/cloud/aiplatform/jobs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 9d60ac6d2b..613295ad70 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -2351,7 +2351,7 @@ def pause(self) -> "ModelDeploymentMonitoringJob": ) else: raise RuntimeError( - "The monitoring job can only be paused under running / pending state, the current state is: %s" + "The monitoring job can only be paused under running state, the current state is: %s" % self.state ) @@ -2368,7 +2368,6 @@ def resume(self) -> "ModelDeploymentMonitoringJob": def delete(self) -> "ModelDeploymentMonitoringJob": """""" - self.pause() self.api_client.delete_model_deployment_monitoring_job( name=self._gca_resource.name ) From d390b69e0bb9cfe9d39947c4e1ca96f036f76959 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Tue, 21 Jun 2022 13:44:23 -0700 Subject: [PATCH 22/53] removing unused parameter --- google/cloud/aiplatform/jobs.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 613295ad70..de46914cd0 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -2075,7 +2075,6 @@ def create( Dict[str, model_monitoring.EndpointObjectiveConfig], ], logging_sampling_strategy: model_monitoring.RandomSampleConfig, - monitor_interval: int, schedule_config: model_monitoring.ScheduleConfig, timeout: float = None, display_name: Optional[str] = None, @@ -2110,12 +2109,6 @@ def create( logging_sampling_strategy (model_monitoring.sampling.RandomSampleConfig): Sample Strategy for logging. - monitor_interval (int): - The model monitoring job scheduling - interval. It will be rounded up to next full - hour. This defines how often the monitoring jobs - are triggered. - schedule_config (model_monitoring.schedule.ScheduleConfig): Configures model monitoring job scheduling interval in hours. This defines how often the monitoring jobs are triggered. From 8b811cc9daf1671f1ec4f049f37eb9bb11ce2c99 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Tue, 28 Jun 2022 06:26:14 +0000 Subject: [PATCH 23/53] addressing some PR comments --- google/cloud/aiplatform/compat/__init__.py | 2 +- google/cloud/aiplatform/jobs.py | 24 ++-- .../aiplatform/model_monitoring/objective.py | 4 +- .../aiplatform/model_monitoring/schedule.py | 4 +- .../unit/aiplatform/test_model_monitoring.py | 112 ++++++++++++++++++ 5 files changed, 129 insertions(+), 17 deletions(-) create mode 100644 tests/unit/aiplatform/test_model_monitoring.py diff --git a/google/cloud/aiplatform/compat/__init__.py b/google/cloud/aiplatform/compat/__init__.py index fcbbeda27e..a31a7b1d4b 100644 --- a/google/cloud/aiplatform/compat/__init__.py +++ b/google/cloud/aiplatform/compat/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2021 Google LLC +# Copyright 2022 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 0e125bc8f0..b139c85a8f 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -1953,7 +1953,6 @@ def __init__( self._gca_resource = self._get_gca_resource( resource_name=model_deployment_monitoring_job_name ) - self._endpoint_resource_name = "" @classmethod def _get_endpoint_resource_name(cls, endpoint: Union[str, "aiplatform.Endpoint"]): @@ -1982,7 +1981,9 @@ def _parse_configs( ], endpoint: Union[str, "aiplatform.Endpoint"], deployed_model_ids: Optional[List[str]] = None, - ): + ) -> List[ + gca_model_deployment_monitoring_job.ModelDeploymentMonitoringObjectiveConfig + ]: """Helper function for matching objective configs with their corresponding models""" all_configs = [] all_models = [] @@ -2001,7 +2002,7 @@ def _parse_configs( response = client.get_endpoint(name=cls._get_endpoint_resource_name(endpoint)) for model in response.deployed_models: all_models.append(model.id) - if model.explanation_spec.parameters == {}: + if model.explanation_spec.parameters != {}: xai_enabled.append(model.id) ## when same objective config is applied to ALL models @@ -2095,27 +2096,28 @@ def create( Args: endpoint (Union[str, "aiplatform.Endpoint"]): - Endpoint resource name. Format: + Required. Endpoint resource name. Format: ``projects/{project}/locations/{location}/endpoints/{endpoint}`` objective_configs (Union[ - model_monitoring.objective.EndpointObjectiveConfig, + Required. model_monitoring.objective.EndpointObjectiveConfig, Dict[str, model_monitoring.objective.EndpointObjectiveConfig]): A single config if it applies to all models, or a dictionary of model_id: model_monitoring.objective.EndpointObjectiveConfig if different model IDs have different configs logging_sampling_strategy (model_monitoring.sampling.RandomSampleConfig): - Sample Strategy for logging. + Required. Sample Strategy for logging. schedule_config (model_monitoring.schedule.ScheduleConfig): - Configures model monitoring job scheduling interval in hours. + Required. Configures model monitoring job scheduling interval in hours. This defines how often the monitoring jobs are triggered. - timeout (float): timeout for the model monitoring job creation request + timeout (float): + Required. Timeout for the model monitoring job creation request display_name (str): - The user-defined name of the + Optional. The user-defined name of the ModelDeploymentMonitoringJob. The name can be up to 128 characters long and can be consist of any UTF-8 characters. @@ -2267,8 +2269,6 @@ def create( "View Model Deployment Monitoring Job:\n%s" % mdm_job._dashboard_uri() ) - mdm_job._endpoint_resource_name = endpoint_resource_name - return mdm_job def update( @@ -2294,7 +2294,7 @@ def update( prediction_drift_detection_config: Optional[ model_monitoring.EndpointDriftDetectionConfig ] = None, - ) -> "ModelDeploymentMonitoringJob": + ) -> None: """""" current_job = self.api_client.get_model_deployment_monitoring_job( name=self._gca_resource.name diff --git a/google/cloud/aiplatform/model_monitoring/objective.py b/google/cloud/aiplatform/model_monitoring/objective.py index a131ce96f5..ef3b850e8a 100644 --- a/google/cloud/aiplatform/model_monitoring/objective.py +++ b/google/cloud/aiplatform/model_monitoring/objective.py @@ -211,9 +211,7 @@ def __init__( input_uri=data_source ) elif data_source.startswith("gs:/"): - training_dataset.gcs_source = gca_io.GcsSource( - uris=[self.skew_detection_config.data_source] - ) + training_dataset.gcs_source = gca_io.GcsSource(uris=[data_source]) if data_format is not None and data_format not in [ "tf-record", "csv", diff --git a/google/cloud/aiplatform/model_monitoring/schedule.py b/google/cloud/aiplatform/model_monitoring/schedule.py index c2f2389530..3d8bae141a 100644 --- a/google/cloud/aiplatform/model_monitoring/schedule.py +++ b/google/cloud/aiplatform/model_monitoring/schedule.py @@ -45,6 +45,8 @@ def __init__(self, monitor_interval: int): def as_proto(self): return ( gca_model_deployment_monitoring_job.ModelDeploymentMonitoringScheduleConfig( - monitor_interval=duration_pb2.Duration(seconds=self.monitor_interval) + monitor_interval=duration_pb2.Duration( + seconds=self.monitor_interval * 3600 + ) ) ) diff --git a/tests/unit/aiplatform/test_model_monitoring.py b/tests/unit/aiplatform/test_model_monitoring.py new file mode 100644 index 0000000000..e2504ad18e --- /dev/null +++ b/tests/unit/aiplatform/test_model_monitoring.py @@ -0,0 +1,112 @@ +# -*- coding: utf-8 -*- + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest + +from google.cloud.aiplatform import model_monitoring + +_TEST_THRESHOLD = 0.1 +_TEST_TARGET_FIELD = "target" +_TEST_BQ_DATASOURCE = "bq://test/data" +_TEST_GCS_DATASOURCE = "gs://test/data" +_TEST_OTHER_DATASOURCE = "" +_TEST_KEY = "key" +_TEST_EMAIL1 = "test1" +_TEST_EMAIL2 = "test2" +_TEST_VALID_DATA_FORMATS = ["tf-record", "csv", "jsonl"] +_TEST_SAMPLING_RATE = 0.8 +_TEST_MONITORING_INTERVAL = 1 + + +class TestModelMonitoringConfigs: + @pytest.mark.parametrize( + "data_source", + [_TEST_BQ_DATASOURCE, _TEST_GCS_DATASOURCE, _TEST_OTHER_DATASOURCE], + ) + @pytest.mark.parametrize("data_format", _TEST_VALID_DATA_FORMATS) + def test_valid_configs(self, data_source, data_format): + random_sample_config = model_monitoring.RandomSampleConfig( + sample_rate=_TEST_SAMPLING_RATE + ) + + schedule_config = model_monitoring.ScheduleConfig( + monitor_interval=_TEST_MONITORING_INTERVAL + ) + + alert_config = model_monitoring.EmailAlertConfig( + user_emails=[_TEST_EMAIL1, _TEST_EMAIL2] + ) + + prediction_drift_config = model_monitoring.EndpointDriftDetectionConfig( + drift_thresholds={_TEST_KEY: _TEST_THRESHOLD} + ) + + skew_config = model_monitoring.EndpointSkewDetectionConfig( + data_source=data_source, + skew_thresholds={_TEST_KEY: _TEST_THRESHOLD}, + target_field=_TEST_TARGET_FIELD, + attribute_skew_thresholds={_TEST_KEY: _TEST_THRESHOLD}, + data_format=data_format, + ) + + xai_config = model_monitoring.EndpointExplanationConfig() + + objective_config = model_monitoring.EndpointObjectiveConfig( + skew_detection_config=skew_config, + drift_detection_config=prediction_drift_config, + explanation_config=xai_config, + ) + + assert ( + objective_config.as_proto().training_dataset == skew_config.training_dataset + ) + assert ( + objective_config.as_proto().training_prediction_skew_detection_config + == skew_config.as_proto() + ) + assert ( + objective_config.as_proto().prediction_drift_detection_config + == prediction_drift_config.as_proto() + ) + assert objective_config.as_proto().explanation_config == xai_config.as_proto() + assert _TEST_EMAIL1 in alert_config.as_proto().email_alert_config.user_emails + assert _TEST_EMAIL2 in alert_config.as_proto().email_alert_config.user_emails + assert ( + random_sample_config.as_proto().random_sample_config.sample_rate + == _TEST_SAMPLING_RATE + ) + assert ( + schedule_config.as_proto().monitor_interval.seconds + == _TEST_MONITORING_INTERVAL * 3600 + ) + + @pytest.mark.parametrize("data_source", [_TEST_GCS_DATASOURCE]) + @pytest.mark.parametrize("data_format", ["other"]) + def test_invalid_data_format(self, data_source, data_format): + if data_format == "other": + with pytest.raises(ValueError) as e: + model_monitoring.EndpointSkewDetectionConfig( + data_source=data_source, + skew_thresholds={_TEST_KEY: _TEST_THRESHOLD}, + target_field=_TEST_TARGET_FIELD, + attribute_skew_thresholds={_TEST_KEY: _TEST_THRESHOLD}, + data_format=data_format, + ) + assert ( + "Unsupported value. `data_format` must be one of 'tf-record', 'csv', or 'jsonl'" + in str(e.value) + ) From 8c5b4d96a039a0e0afaf52fca70615f856816253 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Tue, 28 Jun 2022 06:30:49 +0000 Subject: [PATCH 24/53] docs clarification --- google/cloud/aiplatform/jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index b139c85a8f..77994a4f0f 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -2096,7 +2096,7 @@ def create( Args: endpoint (Union[str, "aiplatform.Endpoint"]): - Required. Endpoint resource name. Format: + Required. Endpoint resource name or an instance of `aiplatform.Endpoint`. Format: ``projects/{project}/locations/{location}/endpoints/{endpoint}`` objective_configs (Union[ From d9f99657fe861734b42cdc2a071e78367e4ae153 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Mon, 27 Jun 2022 23:31:41 -0700 Subject: [PATCH 25/53] Update google/cloud/aiplatform/jobs.py Co-authored-by: sasha-gitg <44654632+sasha-gitg@users.noreply.github.com> --- google/cloud/aiplatform/jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 77994a4f0f..20d99b1dd2 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -2092,7 +2092,7 @@ def create( location: Optional[str] = None, credentials: Optional[auth_credentials.Credentials] = None, ) -> "ModelDeploymentMonitoringJob": - """Creates and launches a model monitoring job + """Creates and launches a model monitoring job. Args: endpoint (Union[str, "aiplatform.Endpoint"]): From deb05129fde5086523b2a01f269ab1e3275fcfaf Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Mon, 27 Jun 2022 23:31:52 -0700 Subject: [PATCH 26/53] Update google/cloud/aiplatform/jobs.py Co-authored-by: sasha-gitg <44654632+sasha-gitg@users.noreply.github.com> --- google/cloud/aiplatform/jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 20d99b1dd2..13f676cdee 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -2104,7 +2104,7 @@ def create( Dict[str, model_monitoring.objective.EndpointObjectiveConfig]): A single config if it applies to all models, or a dictionary of model_id: model_monitoring.objective.EndpointObjectiveConfig if - different model IDs have different configs + different model IDs have different configs. logging_sampling_strategy (model_monitoring.sampling.RandomSampleConfig): Required. Sample Strategy for logging. From 308f4ff351bc9b047b20648c61686a72a7ae0b6e Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Mon, 27 Jun 2022 23:32:07 -0700 Subject: [PATCH 27/53] Update google/cloud/aiplatform/jobs.py Co-authored-by: sasha-gitg <44654632+sasha-gitg@users.noreply.github.com> --- google/cloud/aiplatform/jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 13f676cdee..a8ab856536 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -2123,7 +2123,7 @@ def create( UTF-8 characters. Display name of a ModelDeploymentMonitoringJob. - deployed_model_ids ([List[str]] = ["*"]): + deployed_model_ids (List[str]): Optional. Use this argument to specify which deployed models to apply the objective config to. If left unspecified, the same config will be applied to all deployed models. From eb97eca30692ccc19f189d6338d94b3f384480ae Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Tue, 28 Jun 2022 01:08:41 -0700 Subject: [PATCH 28/53] fixing logic for checking explanation_specs --- google/cloud/aiplatform/jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index a8ab856536..e32d2a647c 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -2002,7 +2002,7 @@ def _parse_configs( response = client.get_endpoint(name=cls._get_endpoint_resource_name(endpoint)) for model in response.deployed_models: all_models.append(model.id) - if model.explanation_spec.parameters != {}: + if str(model.explanation_spec.parameters) != "": xai_enabled.append(model.id) ## when same objective config is applied to ALL models From 96146ac0dcc40b138a18741f7e9dafffb32c0e17 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Tue, 28 Jun 2022 10:38:33 -0700 Subject: [PATCH 29/53] chore: system test for model monitoring (#1428) * chore: adding system test for model monitoring * debugged system test * removing extra print statements * added test coverage for pause resume and delete methods * added test coverage for model monitoring config for two models * fixing test bug * added test coverage for XAI error case --- .../aiplatform/test_model_monitoring.py | 354 ++++++++++++++++++ 1 file changed, 354 insertions(+) create mode 100644 tests/system/aiplatform/test_model_monitoring.py diff --git a/tests/system/aiplatform/test_model_monitoring.py b/tests/system/aiplatform/test_model_monitoring.py new file mode 100644 index 0000000000..f3f1ddfdad --- /dev/null +++ b/tests/system/aiplatform/test_model_monitoring.py @@ -0,0 +1,354 @@ +# -*- coding: utf-8 -*- + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import time + +from google.cloud import aiplatform +from google.cloud.aiplatform.compat.types import job_state as gca_job_state +from google.api_core import exceptions as core_exceptions +from tests.system.aiplatform import e2e_base + +# constants used for testing +USER_EMAIL = "" +MODEL_NAME = "churn" +MODEL_NAME2 = "churn2" +IMAGE = "us-docker.pkg.dev/cloud-aiplatform/prediction/tf2-cpu.2-5:latest" +ENDPOINT = "us-central1-aiplatform.googleapis.com" +CHURN_MODEL_PATH = "gs://mco-mm/churn" +_DEFAULT_INPUT = { + "cnt_ad_reward": 0, + "cnt_challenge_a_friend": 0, + "cnt_completed_5_levels": 1, + "cnt_level_complete_quickplay": 3, + "cnt_level_end_quickplay": 5, + "cnt_level_reset_quickplay": 2, + "cnt_level_start_quickplay": 6, + "cnt_post_score": 34, + "cnt_spend_virtual_currency": 0, + "cnt_use_extra_steps": 0, + "cnt_user_engagement": 120, + "country": "Denmark", + "dayofweek": 3, + "julianday": 254, + "language": "da-dk", + "month": 9, + "operating_system": "IOS", + "user_pseudo_id": "104B0770BAE16E8B53DF330C95881893", +} +JOB_NAME = "churn" + +# Sampling rate (optional, default=.8) +LOG_SAMPLE_RATE = 0.8 + +# Monitoring Interval in seconds (optional, default=3600). +MONITOR_INTERVAL = 3600 + +# URI to training dataset. +DATASET_BQ_URI = "bq://mco-mm.bqmlga4.train" + +# Prediction target column name in training dataset. +TARGET = "churned" + +# Skew and drift thresholds. +DEFAULT_THRESHOLD_VALUE = 0.001 +SKEW_DEFAULT_THRESHOLDS = { + "country": DEFAULT_THRESHOLD_VALUE, + "cnt_user_engagement": DEFAULT_THRESHOLD_VALUE, +} +SKEW_CUSTOM_THRESHOLDS = {"cnt_level_start_quickplay": 0.01} +DRIFT_DEFAULT_THRESHOLDS = { + "country": DEFAULT_THRESHOLD_VALUE, + "cnt_user_engagement": DEFAULT_THRESHOLD_VALUE, +} +DRIFT_CUSTOM_THRESHOLDS = {"cnt_level_start_quickplay": 0.01} +ATTRIB_SKEW_DEFAULT_THRESHOLDS = { + "country": DEFAULT_THRESHOLD_VALUE, + "cnt_user_engagement": DEFAULT_THRESHOLD_VALUE, +} +ATTRIB_SKEW_CUSTOM_THRESHOLDS = {"cnt_level_start_quickplay": 0.01} +ATTRIB_DRIFT_DEFAULT_THRESHOLDS = { + "country": DEFAULT_THRESHOLD_VALUE, + "cnt_user_engagement": DEFAULT_THRESHOLD_VALUE, +} +ATTRIB_DRIFT_CUSTOM_THRESHOLDS = {"cnt_level_start_quickplay": 0.01} + +skew_thresholds = SKEW_DEFAULT_THRESHOLDS.copy() +skew_thresholds.update(SKEW_CUSTOM_THRESHOLDS) +drift_thresholds = DRIFT_DEFAULT_THRESHOLDS.copy() +drift_thresholds.update(DRIFT_CUSTOM_THRESHOLDS) +attrib_skew_thresholds = ATTRIB_SKEW_DEFAULT_THRESHOLDS.copy() +attrib_skew_thresholds.update(ATTRIB_SKEW_CUSTOM_THRESHOLDS) +attrib_drift_thresholds = ATTRIB_DRIFT_DEFAULT_THRESHOLDS.copy() +attrib_drift_thresholds.update(ATTRIB_DRIFT_CUSTOM_THRESHOLDS) + +# global test constants +sampling_strategy = aiplatform.model_monitoring.RandomSampleConfig( + sample_rate=LOG_SAMPLE_RATE +) + +alert_config = aiplatform.model_monitoring.EmailAlertConfig( + user_emails=[USER_EMAIL], enable_logging=True +) + +schedule_config = aiplatform.model_monitoring.ScheduleConfig( + monitor_interval=MONITOR_INTERVAL +) + +skew_config = aiplatform.model_monitoring.EndpointSkewDetectionConfig( + data_source=DATASET_BQ_URI, + skew_thresholds=skew_thresholds, + attribute_skew_thresholds=attrib_skew_thresholds, + target_field=TARGET, +) + +drift_config = aiplatform.model_monitoring.EndpointDriftDetectionConfig( + drift_thresholds=drift_thresholds, + attribute_drift_thresholds=attrib_drift_thresholds, +) + +drift_config2 = aiplatform.model_monitoring.EndpointDriftDetectionConfig( + drift_thresholds=drift_thresholds, + attribute_drift_thresholds=ATTRIB_DRIFT_DEFAULT_THRESHOLDS, +) + +objective_config = aiplatform.model_monitoring.EndpointObjectiveConfig( + skew_config, drift_config +) + +objective_config2 = aiplatform.model_monitoring.EndpointObjectiveConfig( + skew_config, drift_config2 +) + + +@pytest.mark.usefixtures("tear_down_resources") +class TestModelDeploymentMonitoring(e2e_base.TestEndToEnd): + _temp_prefix = "temp_vertex_sdk_e2e_model_monitoring_test" + + def temp_endpoint(self, shared_state): + aiplatform.init( + project=e2e_base._PROJECT, + location=e2e_base._LOCATION, + ) + + model = aiplatform.Model.upload( + display_name=MODEL_NAME, + artifact_uri=CHURN_MODEL_PATH, + serving_container_image_uri=IMAGE, + ) + shared_state["resources"] = [model] + endpoint = model.deploy(machine_type="n1-standard-2") + predict_response = endpoint.predict(instances=[_DEFAULT_INPUT]) + assert len(predict_response.predictions) == 1 + shared_state["resources"].append(endpoint) + return endpoint + + def temp_endpoint_with_two_models(self, shared_state): + aiplatform.init( + project=e2e_base._PROJECT, + location=e2e_base._LOCATION, + ) + + model1 = aiplatform.Model.upload( + display_name=MODEL_NAME, + artifact_uri=CHURN_MODEL_PATH, + serving_container_image_uri=IMAGE, + ) + + model2 = aiplatform.Model.upload( + display_name=MODEL_NAME2, + artifact_uri=CHURN_MODEL_PATH, + serving_container_image_uri=IMAGE, + ) + shared_state["resources"] = [model1, model2] + endpoint = aiplatform.Endpoint.create() + endpoint.deploy( + model=model1, machine_type="n1-standard-2", traffic_percentage=100 + ) + endpoint.deploy( + model=model2, machine_type="n1-standard-2", traffic_percentage=30 + ) + predict_response = endpoint.predict(instances=[_DEFAULT_INPUT]) + assert len(predict_response.predictions) == 1 + shared_state["resources"].append(endpoint) + return endpoint + + def test_mdm_one_model_one_valid_config(self, shared_state): + """ + Upload pre-trained churn model from local file and deploy it for prediction. + """ + # test model monitoring configurations + temp_endpoint = self.temp_endpoint(shared_state) + + job = None + + job = aiplatform.ModelDeploymentMonitoringJob.create( + display_name=JOB_NAME, + logging_sampling_strategy=sampling_strategy, + schedule_config=schedule_config, + alert_config=alert_config, + objective_configs=objective_config, + timeout=3600, + project=e2e_base._PROJECT, + location=e2e_base._LOCATION, + endpoint=temp_endpoint, + predict_instance_schema_uri="", + analysis_instance_schema_uri="", + ) + assert job is not None + + gapic_job = job._gca_resource + assert ( + gapic_job.logging_sampling_strategy.random_sample_config.sample_rate + == LOG_SAMPLE_RATE + ) + assert gapic_job.display_name == JOB_NAME + assert ( + gapic_job.model_deployment_monitoring_schedule_config.monitor_interval.seconds + == MONITOR_INTERVAL + ) + assert ( + gapic_job.model_monitoring_alert_config.email_alert_config.user_emails + == [USER_EMAIL] + ) + assert gapic_job.model_monitoring_alert_config.enable_logging + + gca_obj_config = gapic_job.model_deployment_monitoring_objective_configs[ + 0 + ].objective_config + assert gca_obj_config.training_dataset == skew_config.training_dataset + assert ( + gca_obj_config.training_prediction_skew_detection_config + == skew_config.as_proto() + ) + assert ( + gca_obj_config.prediction_drift_detection_config == drift_config.as_proto() + ) + + job_resource = job._gca_resource.name + + # test job pause, resume, and delete() + timeout = time.time() + 3600 + while time.time() < timeout: + if job.state != gca_job_state.JobState.JOB_STATE_RUNNING: + with pytest.raises(RuntimeError) as e: + job.pause() + assert ( + "The monitoring job can only be paused under running state" + in str(e.value) + ) + if job.state == gca_job_state.JobState.JOB_STATE_RUNNING: + job.pause() + assert job.state == gca_job_state.JobState.JOB_STATE_PAUSED + time.sleep(5) + + while time.time() < timeout: + if job.state == gca_job_state.JobState.JOB_STATE_RUNNING: + break + if job.state == gca_job_state.JobState.JOB_STATE_PAUSED: + job.resume() + time.sleep(5) + job.delete() + with pytest.raises(core_exceptions.NotFound): + job.api_client.get_model_deployment_monitoring_job(name=job_resource) + + def test_mdm_two_models_two_valid_configs(self, shared_state): + temp_endpoint_with_two_models = self.temp_endpoint_with_two_models(shared_state) + [deployed_model1, deployed_model2] = list( + map(lambda x: x.id, temp_endpoint_with_two_models.list_models()) + ) + all_configs = { + deployed_model1: objective_config, + deployed_model2: objective_config2, + } + job = None + job = aiplatform.ModelDeploymentMonitoringJob.create( + display_name=JOB_NAME, + logging_sampling_strategy=sampling_strategy, + schedule_config=schedule_config, + alert_config=alert_config, + objective_configs=all_configs, + timeout=3600, + project=e2e_base._PROJECT, + location=e2e_base._LOCATION, + endpoint=temp_endpoint_with_two_models, + predict_instance_schema_uri="", + analysis_instance_schema_uri="", + ) + assert job is not None + + gapic_job = job._gca_resource + assert ( + gapic_job.logging_sampling_strategy.random_sample_config.sample_rate + == LOG_SAMPLE_RATE + ) + assert gapic_job.display_name == JOB_NAME + assert ( + gapic_job.model_deployment_monitoring_schedule_config.monitor_interval.seconds + == MONITOR_INTERVAL + ) + assert ( + gapic_job.model_monitoring_alert_config.email_alert_config.user_emails + == [USER_EMAIL] + ) + assert gapic_job.model_monitoring_alert_config.enable_logging + + for config in gapic_job.model_deployment_monitoring_objective_configs: + gca_obj_config = config.objective_config + deployed_model_id = config.deployed_model_id + assert ( + gca_obj_config.training_dataset + == all_configs[deployed_model_id].skew_detection_config.training_dataset + ) + assert ( + gca_obj_config.training_prediction_skew_detection_config + == all_configs[deployed_model_id].skew_detection_config.as_proto() + ) + assert ( + gca_obj_config.prediction_drift_detection_config + == all_configs[deployed_model_id].drift_detection_config.as_proto() + ) + + job.delete() + + def test_mdm_invalid_config(self, shared_state): + """ + Upload pre-trained churn model from local file and deploy it for prediction. + """ + # test model monitoring configurations + temp_endpoint = self.temp_endpoint(shared_state) + with pytest.raises(RuntimeError) as e: + objective_config.explanation_config = ( + aiplatform.model_monitoring.EndpointExplanationConfig() + ) + aiplatform.ModelDeploymentMonitoringJob.create( + display_name=JOB_NAME, + logging_sampling_strategy=sampling_strategy, + schedule_config=schedule_config, + alert_config=alert_config, + objective_configs=objective_config, + timeout=3600, + project=e2e_base._PROJECT, + location=e2e_base._LOCATION, + endpoint=temp_endpoint, + predict_instance_schema_uri="", + analysis_instance_schema_uri="", + ) + assert ( + "Invalid config for model. `explanation_config` should only be enabled if the model has `explanation_spec populated" + in str(e.value) + ) From 1a80a8d30d0b313b8347e81f29e686b27a214561 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Tue, 28 Jun 2022 22:06:53 +0000 Subject: [PATCH 30/53] addressed more PR comments --- google/cloud/aiplatform/__init__.py | 16 ++ google/cloud/aiplatform/jobs.py | 254 ++++++++++++------ .../aiplatform/model_monitoring/alert.py | 15 +- .../aiplatform/model_monitoring/objective.py | 2 +- .../aiplatform/model_monitoring/sampling.py | 2 - .../aiplatform/model_monitoring/schedule.py | 2 - .../aiplatform/test_model_monitoring.py | 54 +++- 7 files changed, 225 insertions(+), 120 deletions(-) diff --git a/google/cloud/aiplatform/__init__.py b/google/cloud/aiplatform/__init__.py index f6c6b52da3..7cd41d6625 100644 --- a/google/cloud/aiplatform/__init__.py +++ b/google/cloud/aiplatform/__init__.py @@ -70,6 +70,15 @@ AutoMLTextTrainingJob, AutoMLVideoTrainingJob, ) +from google.cloud.aiplatform.model_monitoring import ( + EmailAlertConfig, + EndpointSkewDetectionConfig, + EndpointDriftDetectionConfig, + EndpointExplanationConfig, + EndpointObjectiveConfig, + RandomSampleConfig, + ScheduleConfig, +) from google.cloud.aiplatform import helpers """ @@ -123,7 +132,12 @@ "CustomTrainingJob", "CustomContainerTrainingJob", "CustomPythonPackageTrainingJob", + "EmailAlertConfig", "Endpoint", + "EndpointDriftDetectionConfig", + "EndpointExplanationConfig", + "EndpointObjectiveConfig", + "EndpointSkewDetectionConfig", "EntityType", "Execution", "Experiment", @@ -138,7 +152,9 @@ "ModelEvaluation", "ModelDeploymentMonitoringJob", "PipelineJob", + "RandomSampleConfig", "SequenceToSequencePlusForecastingTrainingJob", + "ScheduleConfig", "TabularDataset", "Tensorboard", "TensorboardExperiment", diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index e32d2a647c..eb842fc051 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -58,7 +58,6 @@ from google.cloud.aiplatform.utils import console_utils from google.cloud.aiplatform.utils import source_utils from google.cloud.aiplatform.utils import worker_spec_utils -from google.cloud.aiplatform.compat.services import endpoint_service_client _LOGGER = base.Logger(__name__) @@ -1943,7 +1942,23 @@ def __init__( location: Optional[str] = None, credentials: Optional[auth_credentials.Credentials] = None, ): - """Initializer for ModelDeploymentMonitoringJob""" + """Initializer for ModelDeploymentMonitoringJob + + Args: + model_deployment_monitoring_job_name (str): + Required. A fully-qualified ModelDeploymentMonitoringJob resource name or ID. + Example: "projects/.../locations/.../modelDeploymentMonitoringJobs/456" or + "456" when project and location are initialized or passed. + project: Optional[str] = None, + Optional project to retrieve BatchPredictionJob from. If not set, + project set in aiplatform.init will be used. + location: Optional[str] = None, + Optional location to retrieve BatchPredictionJob from. If not set, + location set in aiplatform.init will be used. + credentials: Optional[auth_credentials.Credentials] = None, + Custom credentials to use. If not set, credentials set in + aiplatform.init will be used. + """ super().__init__( job_name=model_deployment_monitoring_job_name, project=project, @@ -1979,66 +1994,58 @@ def _parse_configs( model_monitoring.EndpointObjectiveConfig, Dict[str, model_monitoring.EndpointObjectiveConfig], ], - endpoint: Union[str, "aiplatform.Endpoint"], + endpoint: "aiplatform.Endpoint", deployed_model_ids: Optional[List[str]] = None, ) -> List[ gca_model_deployment_monitoring_job.ModelDeploymentMonitoringObjectiveConfig ]: - """Helper function for matching objective configs with their corresponding models""" - all_configs = [] + """Helper function for matching objective configs with their corresponding models + + Args: + objective_configs (Union[model_monitoring.objective.EndpointObjectiveConfig, + Dict[str, model_monitoring.objective.EndpointObjectiveConfig]): + Required. A single config if it applies to all models, or a dictionary of + model_id: model_monitoring.objective.EndpointObjectiveConfig if + different model IDs have different configs. + endpoint ("aiplatform.Endpoint"): + Required. An instance of aiplatform.Endpoint to launch the MDM job on. + deployed_model_ids (Optional[List[str]]): + Optional. A list of deployed model IDs to apply the objective config to. + Note that a model will have a deployed_model_id that is different from the + uploaded model ID, and IDs in this list should consist of deployed model IDs + on the same endpoint passed in the argument. If `objective_configs` is a dictionary, + then this parameter is ignored. If `objective_configs` is an instance of + `model_monitoring.EndpointObjectiveConfig` and `deployed_model_ids` is a non-empty + list of valid IDs, then the same objective config will apply to all models in this list. + + Returns: + An array of gca_model_deployment_monitoring_job.ModelDeploymentMonitoringObjectiveConfig objects + + Raises: + ValueError, when the model IDs given are invalid + RuntimeError, when XAI is enabled on a model that doesn't have XAI parameters configured + """ all_models = [] xai_enabled = [] - default_endpoint = "aiplatform.googleapis.com" - if aiplatform.initializer.global_config._location is None: - raise ValueError( - "Error parsing model monitoring objective configs: project location is not set" - ) - client_options = dict( - api_endpoint=f"{aiplatform.initializer.global_config._location}-{default_endpoint}" - ) - client = endpoint_service_client.EndpointServiceClient( - client_options=client_options - ) - response = client.get_endpoint(name=cls._get_endpoint_resource_name(endpoint)) - for model in response.deployed_models: + for model in endpoint.list_models(): all_models.append(model.id) if str(model.explanation_spec.parameters) != "": xai_enabled.append(model.id) - ## when same objective config is applied to ALL models - if ( - isinstance(objective_configs, model_monitoring.EndpointObjectiveConfig) - and deployed_model_ids is None - ): - for model in all_models: - if ( - model not in xai_enabled - and objective_configs.explanation_config is not None - ): - raise RuntimeError( - "Invalid config for model. `explanation_config` should only be enabled if the model has `explanation_spec populated" - ) - all_configs.append( - gca_model_deployment_monitoring_job.ModelDeploymentMonitoringObjectiveConfig( - deployed_model_id=model, - objective_config=objective_configs.as_proto(), - ) - ) + all_configs = [] - ## when same objective config is applied to SOME models - elif ( - isinstance(objective_configs, model_monitoring.EndpointObjectiveConfig) - and deployed_model_ids is not None - ): - for model in deployed_model_ids: + ## when same objective config is applied to SOME or ALL models + all_models = deployed_model_ids or all_models + if isinstance(objective_configs, model_monitoring.EndpointObjectiveConfig): + for model in all_models: if ( model not in xai_enabled and objective_configs.explanation_config is not None ): raise RuntimeError( - "Invalid config for model. `explanation_config` should only be enabled if the model has `explanation_spec populated" + "Invalid config for model ID %s. `explanation_config` should only be enabled if the model has `explanation_spec populated" + % model ) - assert model in all_models all_configs.append( gca_model_deployment_monitoring_job.ModelDeploymentMonitoringObjectiveConfig( deployed_model_id=model, @@ -2047,15 +2054,22 @@ def _parse_configs( ) ## when different objective configs are applied to EACH model - elif isinstance(objective_configs, Dict) and deployed_model_ids is None: - assert all(model in all_models for model in objective_configs.keys()) + else: + if not all(model in all_models for model in objective_configs.keys()): + error_string = ( + "Invalid model ID. The model ID must be one of [" + + ",".join(all_models) + + "]. Note that deployed model IDs are different from the uploaded model's ID" + ) + raise ValueError(error_string) for key in objective_configs.keys(): if ( model not in xai_enabled and objective_configs[key].explanation_config is not None ): raise RuntimeError( - "Invalid config for model. `explanation_config` should only be enabled if the model has `explanation_spec populated" + "Invalid config for model ID %s. `explanation_config` should only be enabled if the model has `explanation_spec populated" + % model ) all_configs.append( gca_model_deployment_monitoring_job.ModelDeploymentMonitoringObjectiveConfig( @@ -2076,7 +2090,6 @@ def create( ], logging_sampling_strategy: model_monitoring.RandomSampleConfig, schedule_config: model_monitoring.ScheduleConfig, - timeout: float = None, display_name: Optional[str] = None, deployed_model_ids: Optional[List[str]] = None, alert_config: Optional[model_monitoring.EmailAlertConfig] = None, @@ -2091,6 +2104,7 @@ def create( project: Optional[str] = None, location: Optional[str] = None, credentials: Optional[auth_credentials.Credentials] = None, + create_request_timeout: Optional[float] = None, ) -> "ModelDeploymentMonitoringJob": """Creates and launches a model monitoring job. @@ -2099,10 +2113,9 @@ def create( Required. Endpoint resource name or an instance of `aiplatform.Endpoint`. Format: ``projects/{project}/locations/{location}/endpoints/{endpoint}`` - objective_configs (Union[ - Required. model_monitoring.objective.EndpointObjectiveConfig, + objective_configs (Union[model_monitoring.objective.EndpointObjectiveConfig, Dict[str, model_monitoring.objective.EndpointObjectiveConfig]): - A single config if it applies to all models, or a dictionary of + Required. A single config if it applies to all models, or a dictionary of model_id: model_monitoring.objective.EndpointObjectiveConfig if different model IDs have different configs. @@ -2113,9 +2126,6 @@ def create( Required. Configures model monitoring job scheduling interval in hours. This defines how often the monitoring jobs are triggered. - timeout (float): - Required. Timeout for the model monitoring job creation request - display_name (str): Optional. The user-defined name of the ModelDeploymentMonitoringJob. The name can be up @@ -2190,11 +2200,11 @@ def create( ModelDeploymentMonitoringJob will be secured by this key. + create_request_timeout (int): + Optional. Timeout in seconds for the model monitoring job creation request + Returns: An instance of ModelDeploymentMonitoringJob - - Raises: - ValueError from endpoint resource name and data source path validations """ if not display_name: display_name = cls._generate_display_name() @@ -2214,26 +2224,29 @@ def create( kms_key_name=encryption_spec_key_name ) - mdm_objective_config_seq = cls._parse_configs( - objective_configs, endpoint, deployed_model_ids - ) - - empty_mdm_job = cls._empty_constructor( + self = cls._empty_constructor( project=project, location=location, credentials=credentials, ) parent = initializer.global_config.common_location_path( - project=empty_mdm_job.project, - location=empty_mdm_job.location, + project=self.project, + location=self.location, ) - api_client = empty_mdm_job.api_client + api_client = self.api_client _LOGGER.log_create_with_lro(cls) endpoint_resource_name = cls._get_endpoint_resource_name(endpoint) + + mdm_objective_config_seq = cls._parse_configs( + objective_configs, + aiplatform.Endpoint(endpoint_resource_name), + deployed_model_ids, + ) + gapic_mdm_job = ( gca_model_deployment_monitoring_job.ModelDeploymentMonitoringJob( display_name=display_name, @@ -2255,21 +2268,21 @@ def create( gca_mdm_job = api_client.create_model_deployment_monitoring_job( parent=parent, model_deployment_monitoring_job=gapic_mdm_job, - timeout=timeout, + timeout=create_request_timeout, ) gca_mdm_job._gca_resource = gapic_mdm_job - empty_mdm_job._gca_resource = gca_mdm_job + self._gca_resource = gca_mdm_job - mdm_job = empty_mdm_job + # mdm_job = self - _LOGGER.log_create_complete(cls, mdm_job._gca_resource, "mdm") + _LOGGER.log_create_complete(cls, self._gca_resource, "mdm") _LOGGER.info( - "View Model Deployment Monitoring Job:\n%s" % mdm_job._dashboard_uri() + "View Model Deployment Monitoring Job:\n%s" % self._dashboard_uri() ) - return mdm_job + return self def update( self, @@ -2287,15 +2300,66 @@ def update( ] ] = None, deployed_model_ids: Optional[List[str]] = None, - training_dataset: Optional[str] = None, - training_prediction_skew_detection_config: Optional[ - model_monitoring.EndpointSkewDetectionConfig - ] = None, - prediction_drift_detection_config: Optional[ - model_monitoring.EndpointDriftDetectionConfig - ] = None, ) -> None: - """""" + """Updates an existing ModelDeploymentMonitoringJob + + Args: + + display_name (str): + Optional. The user-defined name of the + ModelDeploymentMonitoringJob. The name can be up + to 128 characters long and can be consist of any + UTF-8 characters. + Display name of a ModelDeploymentMonitoringJob. + + schedule_config (model_monitoring.schedule.ScheduleConfig): + Required. Configures model monitoring job scheduling interval in hours. + This defines how often the monitoring jobs are triggered. + alert_config (model_monitoring.alert.EmailAlertConfig): + Optional. Configures how alerts are sent to the user. Right now + only email alert is supported. + logging_sampling_strategy (model_monitoring.sampling.RandomSampleConfig): + Required. Sample Strategy for logging. + + labels (Dict[str, str]): + Optional. The labels with user-defined metadata to + organize the ModelDeploymentMonitoringJob. + Label keys and values can be no longer than 64 + characters (Unicode codepoints), can only + contain lowercase letters, numeric characters, + underscores and dashes. International characters + are allowed. See https://goo.gl/xmQnxf for more information + and examples of labels. + bigquery_tables_log_ttl (int): + Optional. The TTL(time to live) of BigQuery tables in user projects + which stores logs. A day is the basic unit of + the TTL and we take the ceil of TTL/86400(a + day). e.g. { second: 3600} indicates ttl = 1 + day. + + enable_monitoring_pipeline_logs (bool): + Optional. If true, the scheduled monitoring pipeline logs are sent to + Google Cloud Logging, including pipeline status and + anomalies detected. Please note the logs incur cost, which + are subject to `Cloud Logging + pricing `__. + + objective_configs (Union[ + Required. model_monitoring.objective.EndpointObjectiveConfig, + Dict[str, model_monitoring.objective.EndpointObjectiveConfig]): + A single config if it applies to all models, or a dictionary of + model_id: model_monitoring.objective.EndpointObjectiveConfig if + different model IDs have different configs. + + deployed_model_ids (List[str]): + Optional. Use this argument to specify which deployed models to + apply the updated objective config to. If left unspecified, the same config + will be applied to all deployed models. + + Raises: + RuntimeError, when the job isn't ready to be updated yet + + """ current_job = self.api_client.get_model_deployment_monitoring_job( name=self._gca_resource.name ) @@ -2327,16 +2391,27 @@ def update( update_mask.append("model_deployment_monitoring_objective_configs") current_job.model_deployment_monitoring_objective_configs = ( ModelDeploymentMonitoringJob._parse_configs( - objective_configs, current_job.endpoint, deployed_model_ids + objective_configs, + aiplatform.Endpoint(current_job.endpoint), + deployed_model_ids, ) ) - self.api_client.update_model_deployment_monitoring_job( - model_deployment_monitoring_job=current_job, - update_mask=field_mask_pb2.FieldMask(paths=update_mask), - ) + if self.state == gca_job_state.JobState.JOB_STATE_RUNNING: + self.api_client.update_model_deployment_monitoring_job( + model_deployment_monitoring_job=current_job, + update_mask=field_mask_pb2.FieldMask(paths=update_mask), + ) + else: + raise RuntimeError( + "The monitoring job can only be updated under running state, the current state is: %s" + % self.state + ) def pause(self) -> "ModelDeploymentMonitoringJob": - """""" + """Pause a running MDM job + Raises: + RuntimeError, when the job cannot be paused. + """ if self.state == gca_job_state.JobState.JOB_STATE_RUNNING: self.api_client.pause_model_deployment_monitoring_job( name=self._gca_resource.name @@ -2348,7 +2423,10 @@ def pause(self) -> "ModelDeploymentMonitoringJob": ) def resume(self) -> "ModelDeploymentMonitoringJob": - """""" + """Resumes a paused MDM job + Raises: + RuntimeError, when the job cannot be resumed. + """ if self.state == gca_job_state.JobState.JOB_STATE_PAUSED: self.api_client.resume_model_deployment_monitoring_job( name=self._gca_resource.name @@ -2359,7 +2437,7 @@ def resume(self) -> "ModelDeploymentMonitoringJob": ) def delete(self) -> "ModelDeploymentMonitoringJob": - """""" + """Deletes an MDM job""" self.api_client.delete_model_deployment_monitoring_job( name=self._gca_resource.name ) diff --git a/google/cloud/aiplatform/model_monitoring/alert.py b/google/cloud/aiplatform/model_monitoring/alert.py index f05e87edc7..b165ef9e29 100644 --- a/google/cloud/aiplatform/model_monitoring/alert.py +++ b/google/cloud/aiplatform/model_monitoring/alert.py @@ -15,21 +15,13 @@ # limitations under the License. # -import abc from typing import Optional, List from google.cloud.aiplatform_v1.types import ( model_monitoring as gca_model_monitoring, ) -class _AlertConfig(abc.ABC): - """An abstract class for setting model monitoring alert config""" - - def __init__(self, enable_logging: Optional[bool] = False): - self.enable_logging = enable_logging - - -class EmailAlertConfig(_AlertConfig): +class EmailAlertConfig: def __init__( self, user_emails: List[str] = [], enable_logging: Optional[bool] = False ): @@ -44,11 +36,8 @@ def __init__( [google.cloud.aiplatform.logging.ModelMonitoringAnomaliesLogEntry][]. This can be further sync'd to Pub/Sub or any other services supported by Cloud Logging. - - Returns: - An instance of EmailAlertConfig """ - super().__init__(enable_logging=enable_logging) + self.enable_logging = enable_logging self.user_emails = user_emails def as_proto(self): diff --git a/google/cloud/aiplatform/model_monitoring/objective.py b/google/cloud/aiplatform/model_monitoring/objective.py index ef3b850e8a..cbab46327c 100644 --- a/google/cloud/aiplatform/model_monitoring/objective.py +++ b/google/cloud/aiplatform/model_monitoring/objective.py @@ -191,7 +191,7 @@ def __init__( An instance of EndpointSkewDetectionConfig Raises: - ValueError + ValueError for unsupported data formats """ super().__init__( data_source, diff --git a/google/cloud/aiplatform/model_monitoring/sampling.py b/google/cloud/aiplatform/model_monitoring/sampling.py index c3cd5f47b0..bff967772d 100644 --- a/google/cloud/aiplatform/model_monitoring/sampling.py +++ b/google/cloud/aiplatform/model_monitoring/sampling.py @@ -34,8 +34,6 @@ def __init__(self, sample_rate: Optional[float] = 1): sample_rate (float): Optional. Sets the sampling rate for model monitoring logs. If not set, all logs are processed. - Returns: - An instance of RandomSampleConfig """ super().__init__() self.sample_rate = sample_rate diff --git a/google/cloud/aiplatform/model_monitoring/schedule.py b/google/cloud/aiplatform/model_monitoring/schedule.py index 3d8bae141a..86610828e3 100644 --- a/google/cloud/aiplatform/model_monitoring/schedule.py +++ b/google/cloud/aiplatform/model_monitoring/schedule.py @@ -36,8 +36,6 @@ def __init__(self, monitor_interval: int): monitor_interval (int): Sets the model monitoring job scheduling interval in hours. This defines how often the monitoring jobs are triggered. - Returns: - An instance of ScheduleConfig """ super().__init__() self.monitor_interval = monitor_interval diff --git a/tests/system/aiplatform/test_model_monitoring.py b/tests/system/aiplatform/test_model_monitoring.py index f3f1ddfdad..b28223ef8a 100644 --- a/tests/system/aiplatform/test_model_monitoring.py +++ b/tests/system/aiplatform/test_model_monitoring.py @@ -55,8 +55,8 @@ # Sampling rate (optional, default=.8) LOG_SAMPLE_RATE = 0.8 -# Monitoring Interval in seconds (optional, default=3600). -MONITOR_INTERVAL = 3600 +# Monitoring Interval in hours +MONITOR_INTERVAL = 1 # URI to training dataset. DATASET_BQ_URI = "bq://mco-mm.bqmlga4.train" @@ -202,7 +202,7 @@ def test_mdm_one_model_one_valid_config(self, shared_state): schedule_config=schedule_config, alert_config=alert_config, objective_configs=objective_config, - timeout=3600, + create_request_timeout=3600, project=e2e_base._PROJECT, location=e2e_base._LOCATION, endpoint=temp_endpoint, @@ -219,7 +219,7 @@ def test_mdm_one_model_one_valid_config(self, shared_state): assert gapic_job.display_name == JOB_NAME assert ( gapic_job.model_deployment_monitoring_schedule_config.monitor_interval.seconds - == MONITOR_INTERVAL + == MONITOR_INTERVAL * 3600 ) assert ( gapic_job.model_monitoring_alert_config.email_alert_config.user_emails @@ -241,8 +241,19 @@ def test_mdm_one_model_one_valid_config(self, shared_state): job_resource = job._gca_resource.name - # test job pause, resume, and delete() + # test job update, pause, resume, and delete() timeout = time.time() + 3600 + new_obj_config = aiplatform.model_monitoring.EndpointObjectiveConfig( + skew_config + ) + + while time.time() < timeout: + if job.state == gca_job_state.JobState.JOB_STATE_RUNNING: + job.update(objective_configs=new_obj_config) + print(str(job._gca_resource.prediction_drift_detection_config)) + assert str(job._gca_resource.prediction_drift_detection_config) == "" + break + time.sleep(5) while time.time() < timeout: if job.state != gca_job_state.JobState.JOB_STATE_RUNNING: with pytest.raises(RuntimeError) as e: @@ -282,7 +293,7 @@ def test_mdm_two_models_two_valid_configs(self, shared_state): schedule_config=schedule_config, alert_config=alert_config, objective_configs=all_configs, - timeout=3600, + create_request_timeout=3600, project=e2e_base._PROJECT, location=e2e_base._LOCATION, endpoint=temp_endpoint_with_two_models, @@ -299,7 +310,7 @@ def test_mdm_two_models_two_valid_configs(self, shared_state): assert gapic_job.display_name == JOB_NAME assert ( gapic_job.model_deployment_monitoring_schedule_config.monitor_interval.seconds - == MONITOR_INTERVAL + == MONITOR_INTERVAL * 3600 ) assert ( gapic_job.model_monitoring_alert_config.email_alert_config.user_emails @@ -325,11 +336,26 @@ def test_mdm_two_models_two_valid_configs(self, shared_state): job.delete() - def test_mdm_invalid_config(self, shared_state): - """ - Upload pre-trained churn model from local file and deploy it for prediction. - """ - # test model monitoring configurations + def test_mdm_invalid_config_incorrect_model_id(self, shared_state): + temp_endpoint = self.temp_endpoint(shared_state) + with pytest.raises(ValueError) as e: + aiplatform.ModelDeploymentMonitoringJob.create( + display_name=JOB_NAME, + logging_sampling_strategy=sampling_strategy, + schedule_config=schedule_config, + alert_config=alert_config, + objective_configs=objective_config, + create_request_timeout=3600, + project=e2e_base._PROJECT, + location=e2e_base._LOCATION, + endpoint=temp_endpoint, + predict_instance_schema_uri="", + analysis_instance_schema_uri="", + deployed_model_ids=[""], + ) + assert "Invalid model ID" in str(e.value) + + def test_mdm_invalid_config_xai(self, shared_state): temp_endpoint = self.temp_endpoint(shared_state) with pytest.raises(RuntimeError) as e: objective_config.explanation_config = ( @@ -341,7 +367,7 @@ def test_mdm_invalid_config(self, shared_state): schedule_config=schedule_config, alert_config=alert_config, objective_configs=objective_config, - timeout=3600, + create_request_timeout=3600, project=e2e_base._PROJECT, location=e2e_base._LOCATION, endpoint=temp_endpoint, @@ -349,6 +375,6 @@ def test_mdm_invalid_config(self, shared_state): analysis_instance_schema_uri="", ) assert ( - "Invalid config for model. `explanation_config` should only be enabled if the model has `explanation_spec populated" + "`explanation_config` should only be enabled if the model has `explanation_spec populated" in str(e.value) ) From a3783a2eaaf692869f4e094817b7df21bd9c06f1 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Wed, 29 Jun 2022 19:05:10 +0000 Subject: [PATCH 31/53] addressing remaining comments --- google/cloud/aiplatform/compat/__init__.py | 8 ++-- google/cloud/aiplatform/jobs.py | 51 ++++++++-------------- 2 files changed, 21 insertions(+), 38 deletions(-) diff --git a/google/cloud/aiplatform/compat/__init__.py b/google/cloud/aiplatform/compat/__init__.py index a31a7b1d4b..98ecacc5b5 100644 --- a/google/cloud/aiplatform/compat/__init__.py +++ b/google/cloud/aiplatform/compat/__init__.py @@ -95,9 +95,9 @@ types.model_evaluation = types.model_evaluation_v1beta1 types.model_evaluation_slice = types.model_evaluation_slice_v1beta1 types.model_deployment_monitoring_job = ( - types.model_deployment_monitoring_job_v1beta1, + types.model_deployment_monitoring_job_v1beta1 ) - types.model_monitoring = (types.model_monitoring_v1beta1,) + types.model_monitoring = types.model_monitoring_v1beta1 types.model_service = types.model_service_v1beta1 types.operation = types.operation_v1beta1 types.pipeline_failure_policy = types.pipeline_failure_policy_v1beta1 @@ -183,8 +183,8 @@ types.model = types.model_v1 types.model_evaluation = types.model_evaluation_v1 types.model_evaluation_slice = types.model_evaluation_slice_v1 - types.model_deployment_monitoring_job = (types.model_deployment_monitoring_job_v1,) - types.model_monitoring = (types.model_monitoring_v1,) + types.model_deployment_monitoring_job = types.model_deployment_monitoring_job_v1 + types.model_monitoring = types.model_monitoring_v1 types.model_service = types.model_service_v1 types.operation = types.operation_v1 types.pipeline_failure_policy = types.pipeline_failure_policy_v1 diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 80479d351f..0e2dd7a54c 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -21,7 +21,6 @@ import copy import datetime import time -import re from google.cloud import storage from google.cloud import bigquery @@ -45,11 +44,9 @@ machine_resources as gca_machine_resources_compat, manual_batch_tuning_parameters as gca_manual_batch_tuning_parameters_compat, study as gca_study_compat, + model_deployment_monitoring_job as gca_model_deployment_monitoring_job_compat, ) -from google.cloud.aiplatform_v1.types import ( - model_deployment_monitoring_job as gca_model_deployment_monitoring_job, -) from google.cloud.aiplatform.constants import base as constants from google.cloud.aiplatform import initializer from google.cloud.aiplatform import hyperparameter_tuning @@ -1973,24 +1970,6 @@ def __init__( resource_name=model_deployment_monitoring_job_name ) - @classmethod - def _get_endpoint_resource_name(cls, endpoint: Union[str, "aiplatform.Endpoint"]): - """Helper function for validating endpoint resource name""" - endpoint_resource_string = "" - if isinstance(endpoint, str): - if re.match(r"[0-9]", endpoint): - parent = aiplatform.initializer.global_config.common_location_path() - endpoint_resource_string = f"{parent}/endpoints/{endpoint}" - elif re.match(r"projects/.+?/locations/.+?/endpoints/.+?", endpoint): - endpoint_resource_string = endpoint - elif isinstance(endpoint, aiplatform.Endpoint): - endpoint_resource_string = endpoint.resource_name - else: - raise ValueError( - "Invalid value for endpoint. `endpoint` needs to be one of numeric ID, well-formatted path, or an instance of aiplatform.Endpoint. If providing the full path, it needs to follow the format projects/$PROJECT/locations/$LOCATION/endpoints/$ENDPOINT_ID" - ) - return endpoint_resource_string - @classmethod def _parse_configs( cls, @@ -1998,10 +1977,10 @@ def _parse_configs( model_monitoring.EndpointObjectiveConfig, Dict[str, model_monitoring.EndpointObjectiveConfig], ], - endpoint: "aiplatform.Endpoint", + endpoint: str, deployed_model_ids: Optional[List[str]] = None, ) -> List[ - gca_model_deployment_monitoring_job.ModelDeploymentMonitoringObjectiveConfig + gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringObjectiveConfig ]: """Helper function for matching objective configs with their corresponding models @@ -2011,8 +1990,8 @@ def _parse_configs( Required. A single config if it applies to all models, or a dictionary of model_id: model_monitoring.objective.EndpointObjectiveConfig if different model IDs have different configs. - endpoint ("aiplatform.Endpoint"): - Required. An instance of aiplatform.Endpoint to launch the MDM job on. + endpoint (str): + Required. A valid endpoint resource name to launch the MDM job on. deployed_model_ids (Optional[List[str]]): Optional. A list of deployed model IDs to apply the objective config to. Note that a model will have a deployed_model_id that is different from the @@ -2023,7 +2002,7 @@ def _parse_configs( list of valid IDs, then the same objective config will apply to all models in this list. Returns: - An array of gca_model_deployment_monitoring_job.ModelDeploymentMonitoringObjectiveConfig objects + An array of ModelDeploymentMonitoringObjectiveConfig objects Raises: ValueError, when the model IDs given are invalid @@ -2031,6 +2010,7 @@ def _parse_configs( """ all_models = [] xai_enabled = [] + endpoint = aiplatform.Endpoint(endpoint) for model in endpoint.list_models(): all_models.append(model.id) if str(model.explanation_spec.parameters) != "": @@ -2051,7 +2031,7 @@ def _parse_configs( % model ) all_configs.append( - gca_model_deployment_monitoring_job.ModelDeploymentMonitoringObjectiveConfig( + gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringObjectiveConfig( deployed_model_id=model, objective_config=objective_configs.as_proto(), ) @@ -2076,7 +2056,7 @@ def _parse_configs( % model ) all_configs.append( - gca_model_deployment_monitoring_job.ModelDeploymentMonitoringObjectiveConfig( + gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringObjectiveConfig( deployed_model_id=key, objective_config=objective_configs[key].as_proto(), ) @@ -2243,18 +2223,21 @@ def create( _LOGGER.log_create_with_lro(cls) - endpoint_resource_name = cls._get_endpoint_resource_name(endpoint) + if isinstance(endpoint, str): + endpoint = aiplatform.Endpoint(endpoint).resource_name + else: + endpoint = endpoint.resource_name mdm_objective_config_seq = cls._parse_configs( objective_configs, - aiplatform.Endpoint(endpoint_resource_name), + endpoint, deployed_model_ids, ) gapic_mdm_job = ( - gca_model_deployment_monitoring_job.ModelDeploymentMonitoringJob( + gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringJob( display_name=display_name, - endpoint=endpoint_resource_name, + endpoint=endpoint, model_deployment_monitoring_objective_configs=mdm_objective_config_seq, logging_sampling_strategy=logging_sampling_strategy.as_proto(), model_deployment_monitoring_schedule_config=schedule_config.as_proto(), @@ -2396,7 +2379,7 @@ def update( current_job.model_deployment_monitoring_objective_configs = ( ModelDeploymentMonitoringJob._parse_configs( objective_configs, - aiplatform.Endpoint(current_job.endpoint), + current_job.endpoint, deployed_model_ids, ) ) From aa9c118928813013fa4f2d8bfbb2fca773ccfefb Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Wed, 29 Jun 2022 19:40:29 +0000 Subject: [PATCH 32/53] adding more error checking in parse_configs helper function --- google/cloud/aiplatform/jobs.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 0e2dd7a54c..216473de19 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -2021,6 +2021,13 @@ def _parse_configs( ## when same objective config is applied to SOME or ALL models all_models = deployed_model_ids or all_models if isinstance(objective_configs, model_monitoring.EndpointObjectiveConfig): + if not all(model in all_models for model in objective_configs.keys()): + error_string = ( + "Invalid model ID. The model ID must be one of [" + + ",".join(all_models) + + "]. Note that deployed model IDs are different from the uploaded model's ID" + ) + raise ValueError(error_string) for model in all_models: if ( model not in xai_enabled From b21a0bcfbcf5cf81d5ce2155cf30255cc01dd7e5 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Wed, 29 Jun 2022 19:51:52 +0000 Subject: [PATCH 33/53] minor fix for deployed model ID validation logic --- google/cloud/aiplatform/jobs.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 216473de19..c335759c7c 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -2019,15 +2019,17 @@ def _parse_configs( all_configs = [] ## when same objective config is applied to SOME or ALL models - all_models = deployed_model_ids or all_models + if deployed_model_ids is not None and not all( + model in all_models for model in deployed_model_ids + ): + error_string = ( + "Invalid model ID. The model ID must be one of [" + + ",".join(all_models) + + "]. Note that deployed model IDs are different from the uploaded model's ID" + ) + raise ValueError(error_string) + if isinstance(objective_configs, model_monitoring.EndpointObjectiveConfig): - if not all(model in all_models for model in objective_configs.keys()): - error_string = ( - "Invalid model ID. The model ID must be one of [" - + ",".join(all_models) - + "]. Note that deployed model IDs are different from the uploaded model's ID" - ) - raise ValueError(error_string) for model in all_models: if ( model not in xai_enabled From edbfffa77a8c222cfa746200b5db4ac1091b7591 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Thu, 7 Jul 2022 11:56:36 -0700 Subject: [PATCH 34/53] addressing docs commentes --- google/cloud/aiplatform/jobs.py | 61 +++++++++++++++++---------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index c335759c7c..e0bb52642c 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -114,10 +114,10 @@ def __init__( Example: "projects/123/locations/us-central1/batchPredictionJobs/456" or "456" when project, location and job_type are initialized or passed. project: Optional[str] = None, - Optional project to retrieve Job subclass from. If not set, + Optional. project to retrieve Job subclass from. If not set, project set in aiplatform.init will be used. location: Optional[str] = None, - Optional location to retrieve Job subclass from. If not set, + Optional. location to retrieve Job subclass from. If not set, location set in aiplatform.init will be used. credentials: Optional[auth_credentials.Credentials] = None, Custom credentials to use. If not set, credentials set in @@ -262,7 +262,7 @@ def list( credentials set in aiplatform.init. Returns: - List[VertexAiResourceNoun] - A list of Job resource objects + List[VertexAiResourceNoun] - A list of Job resource objects. """ return cls._list_with_local_order( @@ -311,10 +311,10 @@ def __init__( Example: "projects/.../locations/.../batchPredictionJobs/456" or "456" when project and location are initialized or passed. project: Optional[str] = None, - Optional project to retrieve BatchPredictionJob from. If not set, + Optional. project to retrieve BatchPredictionJob from. If not set, project set in aiplatform.init will be used. location: Optional[str] = None, - Optional location to retrieve BatchPredictionJob from. If not set, + Optional. location to retrieve BatchPredictionJob from. If not set, location set in aiplatform.init will be used. credentials: Optional[auth_credentials.Credentials] = None, Custom credentials to use. If not set, credentials set in @@ -887,7 +887,7 @@ def __init__( Args: project(str): Project of the resource noun. location(str): The location of the resource noun. - credentials(google.auth.credentials.Credentials): Optional custom + credentials(google.auth.credentials.Credentials): Optional. custom credentials to use when accessing interacting with resource noun. """ @@ -1003,10 +1003,10 @@ def get( resource_name (str): Required. A fully-qualified resource name or ID. project (str): - Optional project to retrieve dataset from. If not set, project + Optional. project to retrieve dataset from. If not set, project set in aiplatform.init will be used. location (str): - Optional location to retrieve dataset from. If not set, location + Optional. location to retrieve dataset from. If not set, location set in aiplatform.init will be used. credentials (auth_credentials.Credentials): Custom credentials to use to upload this model. Overrides @@ -1943,21 +1943,21 @@ def __init__( location: Optional[str] = None, credentials: Optional[auth_credentials.Credentials] = None, ): - """Initializer for ModelDeploymentMonitoringJob + """Initializer for ModelDeploymentMonitoringJob. Args: model_deployment_monitoring_job_name (str): Required. A fully-qualified ModelDeploymentMonitoringJob resource name or ID. Example: "projects/.../locations/.../modelDeploymentMonitoringJobs/456" or "456" when project and location are initialized or passed. - project: Optional[str] = None, - Optional project to retrieve BatchPredictionJob from. If not set, + project: (str), + Optional. project to retrieve ModelDeploymentMonitoringJob from. If not set, project set in aiplatform.init will be used. - location: Optional[str] = None, - Optional location to retrieve BatchPredictionJob from. If not set, + location: (str), + Optional. location to retrieve ModelDeploymentMonitoringJob from. If not set, location set in aiplatform.init will be used. - credentials: Optional[auth_credentials.Credentials] = None, - Custom credentials to use. If not set, credentials set in + credentials: (auth_credentials.Credentials), + Optional. Custom credentials to use. If not set, credentials set in aiplatform.init will be used. """ super().__init__( @@ -1982,7 +1982,7 @@ def _parse_configs( ) -> List[ gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringObjectiveConfig ]: - """Helper function for matching objective configs with their corresponding models + """Helper function for matching objective configs with their corresponding models. Args: objective_configs (Union[model_monitoring.objective.EndpointObjectiveConfig, @@ -2002,11 +2002,11 @@ def _parse_configs( list of valid IDs, then the same objective config will apply to all models in this list. Returns: - An array of ModelDeploymentMonitoringObjectiveConfig objects + A List of ModelDeploymentMonitoringObjectiveConfig objects. Raises: - ValueError, when the model IDs given are invalid - RuntimeError, when XAI is enabled on a model that doesn't have XAI parameters configured + ValueError, when the model IDs given are invalid. + RuntimeError, when XAI is enabled on a model that doesn't have XAI parameters configured. """ all_models = [] xai_enabled = [] @@ -2194,10 +2194,10 @@ def create( this key. create_request_timeout (int): - Optional. Timeout in seconds for the model monitoring job creation request + Optional. Timeout in seconds for the model monitoring job creation request. Returns: - An instance of ModelDeploymentMonitoringJob + An instance of ModelDeploymentMonitoringJob. """ if not display_name: display_name = cls._generate_display_name() @@ -2270,8 +2270,6 @@ def create( self._gca_resource = gca_mdm_job - # mdm_job = self - _LOGGER.log_create_complete(cls, self._gca_resource, "mdm") _LOGGER.info( @@ -2296,8 +2294,8 @@ def update( ] ] = None, deployed_model_ids: Optional[List[str]] = None, - ) -> None: - """Updates an existing ModelDeploymentMonitoringJob + ) -> "ModelDeploymentMonitoringJob": + """Updates an existing ModelDeploymentMonitoringJob. Args: @@ -2353,7 +2351,7 @@ def update( will be applied to all deployed models. Raises: - RuntimeError, when the job isn't ready to be updated yet + RuntimeError, when the job isn't ready to be updated yet. """ current_job = self.api_client.get_model_deployment_monitoring_job( @@ -2402,9 +2400,10 @@ def update( "The monitoring job can only be updated under running state, the current state is: %s" % self.state ) + return self def pause(self) -> "ModelDeploymentMonitoringJob": - """Pause a running MDM job + """Pause a running MDM job. Raises: RuntimeError, when the job cannot be paused. """ @@ -2417,9 +2416,10 @@ def pause(self) -> "ModelDeploymentMonitoringJob": "The monitoring job can only be paused under running state, the current state is: %s" % self.state ) + return self def resume(self) -> "ModelDeploymentMonitoringJob": - """Resumes a paused MDM job + """Resumes a paused MDM job. Raises: RuntimeError, when the job cannot be resumed. """ @@ -2431,9 +2431,10 @@ def resume(self) -> "ModelDeploymentMonitoringJob": raise RuntimeError( "The monitoring job can only be resumed under paused state" ) + return self - def delete(self) -> "ModelDeploymentMonitoringJob": - """Deletes an MDM job""" + def delete(self) -> None: + """Deletes an MDM job.""" self.api_client.delete_model_deployment_monitoring_job( name=self._gca_resource.name ) From d07f6f5dfaba25005c2f300ee294fcb9ccc36392 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Thu, 7 Jul 2022 12:19:42 -0700 Subject: [PATCH 35/53] addressed more PR comments --- google/cloud/aiplatform/jobs.py | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index e0bb52642c..6ca4f916b4 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -1977,7 +1977,7 @@ def _parse_configs( model_monitoring.EndpointObjectiveConfig, Dict[str, model_monitoring.EndpointObjectiveConfig], ], - endpoint: str, + endpoint: "aiplatform.Endpoint", deployed_model_ids: Optional[List[str]] = None, ) -> List[ gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringObjectiveConfig @@ -1990,8 +1990,8 @@ def _parse_configs( Required. A single config if it applies to all models, or a dictionary of model_id: model_monitoring.objective.EndpointObjectiveConfig if different model IDs have different configs. - endpoint (str): - Required. A valid endpoint resource name to launch the MDM job on. + endpoint (aiplatform.Endpoint): + Required. A valid instance of aiplatforn.Endpoint to launch the MDM job on. deployed_model_ids (Optional[List[str]]): Optional. A list of deployed model IDs to apply the objective config to. Note that a model will have a deployed_model_id that is different from the @@ -2010,7 +2010,6 @@ def _parse_configs( """ all_models = [] xai_enabled = [] - endpoint = aiplatform.Endpoint(endpoint) for model in endpoint.list_models(): all_models.append(model.id) if str(model.explanation_spec.parameters) != "": @@ -2228,14 +2227,10 @@ def create( location=self.location, ) - api_client = self.api_client - _LOGGER.log_create_with_lro(cls) if isinstance(endpoint, str): - endpoint = aiplatform.Endpoint(endpoint).resource_name - else: - endpoint = endpoint.resource_name + endpoint = aiplatform.Endpoint(endpoint) mdm_objective_config_seq = cls._parse_configs( objective_configs, @@ -2246,7 +2241,7 @@ def create( gapic_mdm_job = ( gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringJob( display_name=display_name, - endpoint=endpoint, + endpoint=endpoint.resource_name, model_deployment_monitoring_objective_configs=mdm_objective_config_seq, logging_sampling_strategy=logging_sampling_strategy.as_proto(), model_deployment_monitoring_schedule_config=schedule_config.as_proto(), @@ -2261,16 +2256,13 @@ def create( ) ) - gca_mdm_job = api_client.create_model_deployment_monitoring_job( + self._gca_resource = self.api_client.create_model_deployment_monitoring_job( parent=parent, model_deployment_monitoring_job=gapic_mdm_job, timeout=create_request_timeout, ) - gca_mdm_job._gca_resource = gapic_mdm_job - self._gca_resource = gca_mdm_job - - _LOGGER.log_create_complete(cls, self._gca_resource, "mdm") + _LOGGER.log_create_complete(cls, self._gca_resource, "mdm_job") _LOGGER.info( "View Model Deployment Monitoring Job:\n%s" % self._dashboard_uri() @@ -2280,6 +2272,7 @@ def create( def update( self, + *, display_name: Optional[str] = None, schedule_config: Optional[model_monitoring.ScheduleConfig] = None, alert_config: Optional[model_monitoring.EmailAlertConfig] = None, @@ -2438,3 +2431,4 @@ def delete(self) -> None: self.api_client.delete_model_deployment_monitoring_job( name=self._gca_resource.name ) + return self From b00e6bc909aa68b9b016bee3c8aedb39c229260c Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Thu, 7 Jul 2022 13:44:09 -0700 Subject: [PATCH 36/53] removing runtime errors --- google/cloud/aiplatform/jobs.py | 24 ++----------------- .../aiplatform/test_model_monitoring.py | 9 +------ 2 files changed, 3 insertions(+), 30 deletions(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 6ca4f916b4..3a2130a715 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -2388,42 +2388,22 @@ def update( model_deployment_monitoring_job=current_job, update_mask=field_mask_pb2.FieldMask(paths=update_mask), ) - else: - raise RuntimeError( - "The monitoring job can only be updated under running state, the current state is: %s" - % self.state - ) return self def pause(self) -> "ModelDeploymentMonitoringJob": - """Pause a running MDM job. - Raises: - RuntimeError, when the job cannot be paused. - """ + """Pause a running MDM job.""" if self.state == gca_job_state.JobState.JOB_STATE_RUNNING: self.api_client.pause_model_deployment_monitoring_job( name=self._gca_resource.name ) - else: - raise RuntimeError( - "The monitoring job can only be paused under running state, the current state is: %s" - % self.state - ) return self def resume(self) -> "ModelDeploymentMonitoringJob": - """Resumes a paused MDM job. - Raises: - RuntimeError, when the job cannot be resumed. - """ + """Resumes a paused MDM job.""" if self.state == gca_job_state.JobState.JOB_STATE_PAUSED: self.api_client.resume_model_deployment_monitoring_job( name=self._gca_resource.name ) - else: - raise RuntimeError( - "The monitoring job can only be resumed under paused state" - ) return self def delete(self) -> None: diff --git a/tests/system/aiplatform/test_model_monitoring.py b/tests/system/aiplatform/test_model_monitoring.py index b28223ef8a..e037fdda28 100644 --- a/tests/system/aiplatform/test_model_monitoring.py +++ b/tests/system/aiplatform/test_model_monitoring.py @@ -250,21 +250,14 @@ def test_mdm_one_model_one_valid_config(self, shared_state): while time.time() < timeout: if job.state == gca_job_state.JobState.JOB_STATE_RUNNING: job.update(objective_configs=new_obj_config) - print(str(job._gca_resource.prediction_drift_detection_config)) assert str(job._gca_resource.prediction_drift_detection_config) == "" break time.sleep(5) while time.time() < timeout: - if job.state != gca_job_state.JobState.JOB_STATE_RUNNING: - with pytest.raises(RuntimeError) as e: - job.pause() - assert ( - "The monitoring job can only be paused under running state" - in str(e.value) - ) if job.state == gca_job_state.JobState.JOB_STATE_RUNNING: job.pause() assert job.state == gca_job_state.JobState.JOB_STATE_PAUSED + break time.sleep(5) while time.time() < timeout: From f1584ac9c711acb84969b8423575f07433daba3e Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Thu, 7 Jul 2022 16:17:54 -0700 Subject: [PATCH 37/53] added more documentation --- .../aiplatform/model_monitoring/objective.py | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/google/cloud/aiplatform/model_monitoring/objective.py b/google/cloud/aiplatform/model_monitoring/objective.py index cbab46327c..e029ac2be7 100644 --- a/google/cloud/aiplatform/model_monitoring/objective.py +++ b/google/cloud/aiplatform/model_monitoring/objective.py @@ -34,7 +34,7 @@ def __init__( attribute_skew_thresholds: Dict[str, float], data_format: Optional[str] = None, ): - """Base class for training-serving skew detection""" + """Base class for training-serving skew detection.""" self.data_source = data_source self.skew_thresholds = skew_thresholds self.attribute_skew_thresholds = attribute_skew_thresholds @@ -67,6 +67,7 @@ def __init__( drift_thresholds: Dict[str, float], attribute_drift_thresholds: Dict[str, float], ): + """Base class for prediction drift detection.""" self.drift_thresholds = drift_thresholds self.attribute_drift_thresholds = attribute_drift_thresholds @@ -148,14 +149,14 @@ def __init__( attribute_skew_thresholds: Optional[Dict[str, float]] = {}, data_format: Optional[str] = None, ): - """Initializer for EndpointSkewDetectionConfig + """Initializer for EndpointSkewDetectionConfig. Args: data_source (str): - Path to training dataset. + Required. Path to training dataset. target_field (str): - The target field name the model is to + Required. The target field name the model is to predict. This field will be excluded when doing Predict and (or) Explain for the training data. @@ -188,10 +189,10 @@ def __init__( The source file is a JSONL file. Returns: - An instance of EndpointSkewDetectionConfig + An instance of EndpointSkewDetectionConfig. Raises: - ValueError for unsupported data formats + ValueError for unsupported data formats. """ super().__init__( data_source, @@ -244,11 +245,21 @@ def __init__( Args: drift_thresholds (Dict[str, float]): + Optional. Key is the feature name and value is the + threshold. If a feature needs to be monitored + for drift, a value threshold must be configured + for that feature. The threshold here is against + feature distribution distance between different + time windws. attribute_drift_thresholds (Dict[str, float]): + Optional. Key is the feature name and value is the + threshold. The threshold here is against + attribution score distance between different + time windows. Returns: - An instance of EndpointDriftDetectionConfig + An instance of EndpointDriftDetectionConfig. """ super().__init__(drift_thresholds, attribute_drift_thresholds) From 6a15b57b3d6f56e60facaaee311de1af75572ce2 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Mon, 11 Jul 2022 20:45:27 +0000 Subject: [PATCH 38/53] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .kokoro/test-samples-impl.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.kokoro/test-samples-impl.sh b/.kokoro/test-samples-impl.sh index 1936350328..8a324c9c7b 100755 --- a/.kokoro/test-samples-impl.sh +++ b/.kokoro/test-samples-impl.sh @@ -33,7 +33,7 @@ export PYTHONUNBUFFERED=1 env | grep KOKORO # Install nox -python3 -m pip install --upgrade --quiet nox +python3.6 -m pip install --upgrade --quiet nox # Use secrets acessor service account to get secrets if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then @@ -76,7 +76,7 @@ for file in samples/**/requirements.txt; do echo "------------------------------------------------------------" # Use nox to execute the tests for the project. - python3 -m nox -s "$RUN_TESTS_SESSION" + python3.6 -m nox -s "$RUN_TESTS_SESSION" EXIT=$? # If this is a periodic build, send the test log to the FlakyBot. From 2e875c34d508187070a675105c412287a1818b20 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 12 Jul 2022 22:22:37 +0000 Subject: [PATCH 39/53] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .kokoro/test-samples-impl.sh | 4 ++-- samples/model-builder/noxfile.py | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.kokoro/test-samples-impl.sh b/.kokoro/test-samples-impl.sh index 8a324c9c7b..1936350328 100755 --- a/.kokoro/test-samples-impl.sh +++ b/.kokoro/test-samples-impl.sh @@ -33,7 +33,7 @@ export PYTHONUNBUFFERED=1 env | grep KOKORO # Install nox -python3.6 -m pip install --upgrade --quiet nox +python3 -m pip install --upgrade --quiet nox # Use secrets acessor service account to get secrets if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then @@ -76,7 +76,7 @@ for file in samples/**/requirements.txt; do echo "------------------------------------------------------------" # Use nox to execute the tests for the project. - python3.6 -m nox -s "$RUN_TESTS_SESSION" + python3 -m nox -s "$RUN_TESTS_SESSION" EXIT=$? # If this is a periodic build, send the test log to the FlakyBot. diff --git a/samples/model-builder/noxfile.py b/samples/model-builder/noxfile.py index a40410b563..38bb0a572b 100644 --- a/samples/model-builder/noxfile.py +++ b/samples/model-builder/noxfile.py @@ -180,7 +180,6 @@ def blacken(session: nox.sessions.Session) -> None: # format = isort + black # - @nox.session def format(session: nox.sessions.Session) -> None: """ @@ -230,7 +229,9 @@ def _session_tests( if os.path.exists("requirements-test.txt"): if os.path.exists("constraints-test.txt"): - session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt") + session.install( + "-r", "requirements-test.txt", "-c", "constraints-test.txt" + ) else: session.install("-r", "requirements-test.txt") with open("requirements-test.txt") as rtfile: @@ -243,9 +244,9 @@ def _session_tests( post_install(session) if "pytest-parallel" in packages: - concurrent_args.extend(["--workers", "auto", "--tests-per-worker", "auto"]) + concurrent_args.extend(['--workers', 'auto', '--tests-per-worker', 'auto']) elif "pytest-xdist" in packages: - concurrent_args.extend(["-n", "auto"]) + concurrent_args.extend(['-n', 'auto']) session.run( "pytest", @@ -275,7 +276,7 @@ def py(session: nox.sessions.Session) -> None: def _get_repo_root() -> Optional[str]: - """Returns the root folder of the project.""" + """ Returns the root folder of the project. """ # Get root of this repository. Assume we don't have directories nested deeper than 10 items. p = Path(os.getcwd()) for i in range(10): From 500fd0427b5e2f9c8f52a6baa7d70b5aa3abbfa7 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Fri, 15 Jul 2022 17:23:59 -0700 Subject: [PATCH 40/53] addressing PR comments --- google/cloud/aiplatform/__init__.py | 10 +--- google/cloud/aiplatform/jobs.py | 57 +++++++++++-------- .../aiplatform/test_model_monitoring.py | 21 +++---- 3 files changed, 44 insertions(+), 44 deletions(-) diff --git a/google/cloud/aiplatform/__init__.py b/google/cloud/aiplatform/__init__.py index b897b57baa..bb9fdfecbc 100644 --- a/google/cloud/aiplatform/__init__.py +++ b/google/cloud/aiplatform/__init__.py @@ -71,15 +71,7 @@ AutoMLTextTrainingJob, AutoMLVideoTrainingJob, ) -from google.cloud.aiplatform.model_monitoring import ( - EmailAlertConfig, - EndpointSkewDetectionConfig, - EndpointDriftDetectionConfig, - EndpointExplanationConfig, - EndpointObjectiveConfig, - RandomSampleConfig, - ScheduleConfig, -) + from google.cloud.aiplatform import helpers """ diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 7de98bf86b..c362e540cc 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -1930,7 +1930,7 @@ class ModelDeploymentMonitoringJob(_Job): _resource_noun = "modelDeploymentMonitoringJobs" _getter_method = "get_model_deployment_monitoring_job" _list_method = "list_model_deployment_monitoring_jobs" - _cancel_method = None + _cancel_method = "cancel_model_deployment_monitoring_jobs" _delete_method = "delete_model_deployment_monitoring_job" _job_type = "model-deployment-monitoring" _parse_resource_name_method = "parse_model_deployment_monitoring_job_path" @@ -2076,12 +2076,12 @@ def _parse_configs( def create( cls, endpoint: Union[str, "aiplatform.Endpoint"], - objective_configs: Union[ + objective_configs: Optional[Union[ model_monitoring.EndpointObjectiveConfig, Dict[str, model_monitoring.EndpointObjectiveConfig], - ], - logging_sampling_strategy: model_monitoring.RandomSampleConfig, - schedule_config: model_monitoring.ScheduleConfig, + ]] = None, + logging_sampling_strategy: Optional[model_monitoring.RandomSampleConfig] = None, + schedule_config: Optional[model_monitoring.ScheduleConfig] = None, display_name: Optional[str] = None, deployed_model_ids: Optional[List[str]] = None, alert_config: Optional[model_monitoring.EmailAlertConfig] = None, @@ -2106,16 +2106,16 @@ def create( ``projects/{project}/locations/{location}/endpoints/{endpoint}`` objective_configs (Union[model_monitoring.objective.EndpointObjectiveConfig, - Dict[str, model_monitoring.objective.EndpointObjectiveConfig]): + Optional. Dict[str, model_monitoring.objective.EndpointObjectiveConfig]): Required. A single config if it applies to all models, or a dictionary of model_id: model_monitoring.objective.EndpointObjectiveConfig if different model IDs have different configs. logging_sampling_strategy (model_monitoring.sampling.RandomSampleConfig): - Required. Sample Strategy for logging. + Optional. Sample Strategy for logging. schedule_config (model_monitoring.schedule.ScheduleConfig): - Required. Configures model monitoring job scheduling interval in hours. + Optional. Configures model monitoring job scheduling interval in hours. This defines how often the monitoring jobs are triggered. display_name (str): @@ -2216,10 +2216,12 @@ def create( kms_key_name=encryption_spec_key_name ) + if credentials is None and isinstance(endpoint, aiplatform.Endpoint): + credentials = endpoint.credentials self = cls._empty_constructor( project=project, location=location, - credentials=credentials, + credentials=credentials ) parent = initializer.global_config.common_location_path( @@ -2227,10 +2229,8 @@ def create( location=self.location, ) - _LOGGER.log_create_with_lro(cls) - if isinstance(endpoint, str): - endpoint = aiplatform.Endpoint(endpoint) + endpoint = aiplatform.Endpoint(endpoint, project, location, credentials) mdm_objective_config_seq = cls._parse_configs( objective_configs, @@ -2256,6 +2256,7 @@ def create( ) ) + _LOGGER.log_create_with_lro(cls) self._gca_resource = self.api_client.create_model_deployment_monitoring_job( parent=parent, model_deployment_monitoring_job=gapic_mdm_job, @@ -2270,6 +2271,16 @@ def create( return self + @classmethod + def cancel(cls): + raise NotImplementedError( + "Cancel method is not implemented because it is not applicable. A running model deployment monitoring job can be paused or deleted." + ) + @property + def end_time(self): + _LOGGER.info("Model deployment monitoring jobs do not have an end time since their inactive states are either PAUSED or PENDING.") + return None + def update( self, *, @@ -2342,39 +2353,36 @@ def update( Optional. Use this argument to specify which deployed models to apply the updated objective config to. If left unspecified, the same config will be applied to all deployed models. - - Raises: - RuntimeError, when the job isn't ready to be updated yet. - """ + self._sync_gca_resource() current_job = self.api_client.get_model_deployment_monitoring_job( name=self._gca_resource.name ) update_mask: List[str] = [] - if display_name: + if display_name is not None: update_mask.append("display_name") current_job.display_name = display_name - if schedule_config: + if schedule_config is not None: update_mask.append("model_deployment_monitoring_schedule_config") current_job.model_deployment_monitoring_schedule_config = schedule_config - if alert_config: + if alert_config is not None: update_mask.append("model_monitoring_alert_config") current_job.model_monitoring_alert_config = alert_config - if logging_sampling_strategy: + if logging_sampling_strategy is not None: update_mask.append("logging_sampling_strategy") current_job.logging_sampling_strategy = logging_sampling_strategy - if labels: + if labels is not None: update_mask.append("labels") current_job.lables = labels - if bigquery_tables_log_ttl: + if bigquery_tables_log_ttl is not None: update_mask.append("log_ttl") current_job.log_ttl = bigquery_tables_log_ttl - if enable_monitoring_pipeline_logs: + if enable_monitoring_pipeline_logs is not None: update_mask.append("enable_monitoring_pipeline_logs") current_job.enable_monitoring_pipeline_logs = ( enable_monitoring_pipeline_logs ) - if objective_configs: + if objective_configs is not None: update_mask.append("model_deployment_monitoring_objective_configs") current_job.model_deployment_monitoring_objective_configs = ( ModelDeploymentMonitoringJob._parse_configs( @@ -2411,4 +2419,3 @@ def delete(self) -> None: self.api_client.delete_model_deployment_monitoring_job( name=self._gca_resource.name ) - return self diff --git a/tests/system/aiplatform/test_model_monitoring.py b/tests/system/aiplatform/test_model_monitoring.py index e037fdda28..3ef9cdd456 100644 --- a/tests/system/aiplatform/test_model_monitoring.py +++ b/tests/system/aiplatform/test_model_monitoring.py @@ -19,6 +19,7 @@ import time from google.cloud import aiplatform +from google.cloud.aiplatform import model_monitoring from google.cloud.aiplatform.compat.types import job_state as gca_job_state from google.api_core import exceptions as core_exceptions from tests.system.aiplatform import e2e_base @@ -97,40 +98,40 @@ attrib_drift_thresholds.update(ATTRIB_DRIFT_CUSTOM_THRESHOLDS) # global test constants -sampling_strategy = aiplatform.model_monitoring.RandomSampleConfig( +sampling_strategy = model_monitoring.RandomSampleConfig( sample_rate=LOG_SAMPLE_RATE ) -alert_config = aiplatform.model_monitoring.EmailAlertConfig( +alert_config = model_monitoring.EmailAlertConfig( user_emails=[USER_EMAIL], enable_logging=True ) -schedule_config = aiplatform.model_monitoring.ScheduleConfig( +schedule_config = model_monitoring.ScheduleConfig( monitor_interval=MONITOR_INTERVAL ) -skew_config = aiplatform.model_monitoring.EndpointSkewDetectionConfig( +skew_config = model_monitoring.EndpointSkewDetectionConfig( data_source=DATASET_BQ_URI, skew_thresholds=skew_thresholds, attribute_skew_thresholds=attrib_skew_thresholds, target_field=TARGET, ) -drift_config = aiplatform.model_monitoring.EndpointDriftDetectionConfig( +drift_config = model_monitoring.EndpointDriftDetectionConfig( drift_thresholds=drift_thresholds, attribute_drift_thresholds=attrib_drift_thresholds, ) -drift_config2 = aiplatform.model_monitoring.EndpointDriftDetectionConfig( +drift_config2 = model_monitoring.EndpointDriftDetectionConfig( drift_thresholds=drift_thresholds, attribute_drift_thresholds=ATTRIB_DRIFT_DEFAULT_THRESHOLDS, ) -objective_config = aiplatform.model_monitoring.EndpointObjectiveConfig( +objective_config = model_monitoring.EndpointObjectiveConfig( skew_config, drift_config ) -objective_config2 = aiplatform.model_monitoring.EndpointObjectiveConfig( +objective_config2 = model_monitoring.EndpointObjectiveConfig( skew_config, drift_config2 ) @@ -243,7 +244,7 @@ def test_mdm_one_model_one_valid_config(self, shared_state): # test job update, pause, resume, and delete() timeout = time.time() + 3600 - new_obj_config = aiplatform.model_monitoring.EndpointObjectiveConfig( + new_obj_config = model_monitoring.EndpointObjectiveConfig( skew_config ) @@ -352,7 +353,7 @@ def test_mdm_invalid_config_xai(self, shared_state): temp_endpoint = self.temp_endpoint(shared_state) with pytest.raises(RuntimeError) as e: objective_config.explanation_config = ( - aiplatform.model_monitoring.EndpointExplanationConfig() + model_monitoring.EndpointExplanationConfig() ) aiplatform.ModelDeploymentMonitoringJob.create( display_name=JOB_NAME, From c44bd16fcbfab3b797d5309fd5210213b199c410 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Sat, 16 Jul 2022 00:26:45 +0000 Subject: [PATCH 41/53] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- google/cloud/aiplatform/jobs.py | 19 ++++++++++-------- .../aiplatform/test_model_monitoring.py | 20 +++++-------------- 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index c362e540cc..1039b95cc4 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -2076,10 +2076,12 @@ def _parse_configs( def create( cls, endpoint: Union[str, "aiplatform.Endpoint"], - objective_configs: Optional[Union[ - model_monitoring.EndpointObjectiveConfig, - Dict[str, model_monitoring.EndpointObjectiveConfig], - ]] = None, + objective_configs: Optional[ + Union[ + model_monitoring.EndpointObjectiveConfig, + Dict[str, model_monitoring.EndpointObjectiveConfig], + ] + ] = None, logging_sampling_strategy: Optional[model_monitoring.RandomSampleConfig] = None, schedule_config: Optional[model_monitoring.ScheduleConfig] = None, display_name: Optional[str] = None, @@ -2219,9 +2221,7 @@ def create( if credentials is None and isinstance(endpoint, aiplatform.Endpoint): credentials = endpoint.credentials self = cls._empty_constructor( - project=project, - location=location, - credentials=credentials + project=project, location=location, credentials=credentials ) parent = initializer.global_config.common_location_path( @@ -2276,9 +2276,12 @@ def cancel(cls): raise NotImplementedError( "Cancel method is not implemented because it is not applicable. A running model deployment monitoring job can be paused or deleted." ) + @property def end_time(self): - _LOGGER.info("Model deployment monitoring jobs do not have an end time since their inactive states are either PAUSED or PENDING.") + _LOGGER.info( + "Model deployment monitoring jobs do not have an end time since their inactive states are either PAUSED or PENDING." + ) return None def update( diff --git a/tests/system/aiplatform/test_model_monitoring.py b/tests/system/aiplatform/test_model_monitoring.py index 3ef9cdd456..3551a7641c 100644 --- a/tests/system/aiplatform/test_model_monitoring.py +++ b/tests/system/aiplatform/test_model_monitoring.py @@ -98,17 +98,13 @@ attrib_drift_thresholds.update(ATTRIB_DRIFT_CUSTOM_THRESHOLDS) # global test constants -sampling_strategy = model_monitoring.RandomSampleConfig( - sample_rate=LOG_SAMPLE_RATE -) +sampling_strategy = model_monitoring.RandomSampleConfig(sample_rate=LOG_SAMPLE_RATE) alert_config = model_monitoring.EmailAlertConfig( user_emails=[USER_EMAIL], enable_logging=True ) -schedule_config = model_monitoring.ScheduleConfig( - monitor_interval=MONITOR_INTERVAL -) +schedule_config = model_monitoring.ScheduleConfig(monitor_interval=MONITOR_INTERVAL) skew_config = model_monitoring.EndpointSkewDetectionConfig( data_source=DATASET_BQ_URI, @@ -127,13 +123,9 @@ attribute_drift_thresholds=ATTRIB_DRIFT_DEFAULT_THRESHOLDS, ) -objective_config = model_monitoring.EndpointObjectiveConfig( - skew_config, drift_config -) +objective_config = model_monitoring.EndpointObjectiveConfig(skew_config, drift_config) -objective_config2 = model_monitoring.EndpointObjectiveConfig( - skew_config, drift_config2 -) +objective_config2 = model_monitoring.EndpointObjectiveConfig(skew_config, drift_config2) @pytest.mark.usefixtures("tear_down_resources") @@ -244,9 +236,7 @@ def test_mdm_one_model_one_valid_config(self, shared_state): # test job update, pause, resume, and delete() timeout = time.time() + 3600 - new_obj_config = model_monitoring.EndpointObjectiveConfig( - skew_config - ) + new_obj_config = model_monitoring.EndpointObjectiveConfig(skew_config) while time.time() < timeout: if job.state == gca_job_state.JobState.JOB_STATE_RUNNING: From e09312abad4130d0acf57e0f228b24e526b4f0f6 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Mon, 18 Jul 2022 16:29:23 -0700 Subject: [PATCH 42/53] addressing more PR comments --- google/cloud/aiplatform/jobs.py | 4 +- .../aiplatform/model_monitoring/alert.py | 1 + .../aiplatform/model_monitoring/objective.py | 148 +++++++++++++----- .../aiplatform/model_monitoring/sampling.py | 10 +- .../aiplatform/model_monitoring/schedule.py | 9 +- 5 files changed, 115 insertions(+), 57 deletions(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 1039b95cc4..7fa0baa15b 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -2107,8 +2107,8 @@ def create( Required. Endpoint resource name or an instance of `aiplatform.Endpoint`. Format: ``projects/{project}/locations/{location}/endpoints/{endpoint}`` - objective_configs (Union[model_monitoring.objective.EndpointObjectiveConfig, - Optional. Dict[str, model_monitoring.objective.EndpointObjectiveConfig]): + objective_configs (Union[model_monitoring.EndpointObjectiveConfig, + Dict[str, model_monitoring.EndpointObjectiveConfig]]): Required. A single config if it applies to all models, or a dictionary of model_id: model_monitoring.objective.EndpointObjectiveConfig if different model IDs have different configs. diff --git a/google/cloud/aiplatform/model_monitoring/alert.py b/google/cloud/aiplatform/model_monitoring/alert.py index b165ef9e29..43e62afcfa 100644 --- a/google/cloud/aiplatform/model_monitoring/alert.py +++ b/google/cloud/aiplatform/model_monitoring/alert.py @@ -41,6 +41,7 @@ def __init__( self.user_emails = user_emails def as_proto(self): + """Returns EmailAlertConfig as a proto message.""" user_email_alert_config = ( gca_model_monitoring.ModelMonitoringAlertConfig.EmailAlertConfig( user_emails=self.user_emails diff --git a/google/cloud/aiplatform/model_monitoring/objective.py b/google/cloud/aiplatform/model_monitoring/objective.py index e029ac2be7..a92b781c4d 100644 --- a/google/cloud/aiplatform/model_monitoring/objective.py +++ b/google/cloud/aiplatform/model_monitoring/objective.py @@ -24,6 +24,9 @@ model_monitoring as gca_model_monitoring, ) +TF_RECORD = "tf-record" +CSV = "CSV" +JSONL = "jsonl" class _SkewDetectionConfig(abc.ABC): def __init__( @@ -34,7 +37,46 @@ def __init__( attribute_skew_thresholds: Dict[str, float], data_format: Optional[str] = None, ): - """Base class for training-serving skew detection.""" + """Base class for training-serving skew detection. + Args: + data_source (str): + Required. Path to training dataset. + + skew_thresholds (Dict[str, float]): + Optional. Key is the feature name and value is the + threshold. If a feature needs to be monitored + for skew, a value threshold must be configured + for that feature. The threshold here is against + feature distribution distance between the + training and prediction feature. + + target_field (str): + Required. The target field name the model is to + predict. This field will be excluded when doing + Predict and (or) Explain for the training data. + + attribute_skew_thresholds (Dict[str, float]): + Optional. Key is the feature name and value is the + threshold. Feature attributions indicate how much + each feature in your model contributed to the + predictions for each given instance. + + data_format (str): + Optional. Data format of the dataset, only applicable + if the input is from Google Cloud Storage. + The possible formats are: + + "tf-record" + The source file is a TFRecord file. + + "csv" + The source file is a CSV file. + + "jsonl" + The source file is a JSONL file. + + + """ self.data_source = data_source self.skew_thresholds = skew_thresholds self.attribute_skew_thresholds = attribute_skew_thresholds @@ -43,18 +85,21 @@ def __init__( self.training_dataset = None def as_proto(self): + """Returns _SkewDetectionConfig as a proto message.""" skew_thresholds_mapping = {} attribution_score_skew_thresholds_mapping = {} - for key in self.skew_thresholds.keys(): - skew_threshold = gca_threshold_config(value=self.skew_thresholds[key]) - skew_thresholds_mapping[key] = skew_threshold - for key in self.attribute_skew_thresholds.keys(): - attribution_score_skew_threshold = gca_threshold_config( - value=self.attribute_skew_thresholds[key] - ) - attribution_score_skew_thresholds_mapping[ - key - ] = attribution_score_skew_threshold + if self.skew_thresholds is not None: + for key in self.skew_thresholds.keys(): + skew_threshold = gca_threshold_config(value=self.skew_thresholds[key]) + skew_thresholds_mapping[key] = skew_threshold + if self.attribute_skew_thresholds is not None: + for key in self.attribute_skew_thresholds.keys(): + attribution_score_skew_threshold = gca_threshold_config( + value=self.attribute_skew_thresholds[key] + ) + attribution_score_skew_thresholds_mapping[ + key + ] = attribution_score_skew_threshold return gca_model_monitoring.ModelMonitoringObjectiveConfig.TrainingPredictionSkewDetectionConfig( skew_thresholds=skew_thresholds_mapping, attribution_score_skew_thresholds=attribution_score_skew_thresholds_mapping, @@ -67,23 +112,40 @@ def __init__( drift_thresholds: Dict[str, float], attribute_drift_thresholds: Dict[str, float], ): - """Base class for prediction drift detection.""" + """Base class for prediction drift detection. + Args: + drift_thresholds (Dict[str, float]): + Required. Key is the feature name and value is the + threshold. If a feature needs to be monitored + for drift, a value threshold must be configured + for that feature. The threshold here is against + feature distribution distance between different + time windws. + attribute_drift_thresholds (Dict[str, float]): + Required. Key is the feature name and value is the + threshold. The threshold here is against + attribution score distance between different + time windows. + """ self.drift_thresholds = drift_thresholds self.attribute_drift_thresholds = attribute_drift_thresholds def as_proto(self): + """Returns drift detection config as a proto message.""" drift_thresholds_mapping = {} attribution_score_drift_thresholds_mapping = {} - for key in self.drift_thresholds.keys(): - drift_threshold = gca_threshold_config(value=self.drift_thresholds[key]) - drift_thresholds_mapping[key] = drift_threshold - for key in self.attribute_drift_thresholds.keys(): - attribution_score_drift_threshold = gca_threshold_config( - value=self.attribute_drift_thresholds[key] - ) - attribution_score_drift_thresholds_mapping[ - key - ] = attribution_score_drift_threshold + if self.drift_thresholds is not None: + for key in self.drift_thresholds.keys(): + drift_threshold = gca_threshold_config(value=self.drift_thresholds[key]) + drift_thresholds_mapping[key] = drift_threshold + if self.attribute_drift_thresholds is not None: + for key in self.attribute_drift_thresholds.keys(): + attribution_score_drift_threshold = gca_threshold_config( + value=self.attribute_drift_thresholds[key] + ) + attribution_score_drift_thresholds_mapping[ + key + ] = attribution_score_drift_threshold return gca_model_monitoring.ModelMonitoringObjectiveConfig.PredictionDriftDetectionConfig( drift_thresholds=drift_thresholds_mapping, attribution_score_drift_thresholds=attribution_score_drift_thresholds_mapping, @@ -92,9 +154,11 @@ def as_proto(self): class _ExplanationConfig(abc.ABC): def __init__(self): + """Base class for EndpointExplanationConfig.""" self.enable_feature_attributes = False def as_proto(self): + """Returns _ExplanationConfig as a proto message""" return gca_model_monitoring.ModelMonitoringObjectiveConfig.ExplanationConfig( enable_feature_attributes=self.enable_feature_attributes ) @@ -116,6 +180,7 @@ def __init__( self.explanation_config = explanation_config def as_proto(self): + """Returns _ObjectiveConfig as a proto message.""" training_dataset = None if self.skew_detection_config is not None: training_dataset = self.skew_detection_config.training_dataset @@ -145,8 +210,8 @@ def __init__( self, data_source: str, target_field: str, - skew_thresholds: Optional[Dict[str, float]] = {}, - attribute_skew_thresholds: Optional[Dict[str, float]] = {}, + skew_thresholds: Optional[Dict[str, float]] = None, + attribute_skew_thresholds: Optional[Dict[str, float]] = None, data_format: Optional[str] = None, ): """Initializer for EndpointSkewDetectionConfig. @@ -188,9 +253,6 @@ def __init__( "jsonl" The source file is a JSONL file. - Returns: - An instance of EndpointSkewDetectionConfig. - Raises: ValueError for unsupported data formats. """ @@ -214,12 +276,10 @@ def __init__( elif data_source.startswith("gs:/"): training_dataset.gcs_source = gca_io.GcsSource(uris=[data_source]) if data_format is not None and data_format not in [ - "tf-record", - "csv", - "jsonl", + TF_RECORD, CSV, JSONL ]: raise ValueError( - "Unsupported value. `data_format` must be one of 'tf-record', 'csv', or 'jsonl'" + "Unsupported value. `data_format` must be one of %s, %s, or %s" % (TF_RECORD, CSV, JSONL) ) training_dataset.data_format = data_format else: @@ -238,8 +298,8 @@ class EndpointDriftDetectionConfig(_DriftDetectionConfig): def __init__( self, - drift_thresholds: Optional[Dict[str, float]] = {}, - attribute_drift_thresholds: Optional[Dict[str, float]] = {}, + drift_thresholds: Optional[Dict[str, float]] = None, + attribute_drift_thresholds: Optional[Dict[str, float]] = None, ): """Initializer for EndpointDriftDetectionConfig @@ -257,9 +317,6 @@ def __init__( threshold. The threshold here is against attribution score distance between different time windows. - - Returns: - An instance of EndpointDriftDetectionConfig. """ super().__init__(drift_thresholds, attribute_drift_thresholds) @@ -267,10 +324,11 @@ def __init__( class EndpointExplanationConfig(_ExplanationConfig): """A class that enables Vertex Explainable AI. - Only applicable if the model has explanation_spec populated. + Only applicable if the model has explanation_spec populated. By default, explanation config is disabled. Instantiating this class will enable the config. """ def __init__(self): + """Initializer for EndpointExplanationConfig.""" super().__init__() self.enable_feature_attributes = True @@ -281,16 +339,24 @@ class EndpointObjectiveConfig(_ObjectiveConfig): def __init__( self, skew_detection_config: Optional[ - "gca_model_monitoring.EndpointSkewDetectionConfig" + "EndpointSkewDetectionConfig" ] = None, drift_detection_config: Optional[ - "gca_model_monitoring.EndpointDriftDetectionConfig" + "EndpointDriftDetectionConfig" ] = None, explanation_config: Optional[ - "gca_model_monitoring.EndpointExplanationConfig" + "EndpointExplanationConfig" ] = None, ): - """""" + """Initializer for EndpointObjectiveConfig. + Args: + skew_detection_config (EndpointSkewDetectionConfig): + Optional. An instance of EndpointSkewDetectionConfig. + drift_detection_config (EndpointDriftDetectionConfig): + Optional. An instance of EndpointDriftDetectionConfig. + explanation_config (EndpointExplanationConfig): + Optional. An instance of EndpointExplanationConfig. + """ super().__init__( skew_detection_config, drift_detection_config, explanation_config ) diff --git a/google/cloud/aiplatform/model_monitoring/sampling.py b/google/cloud/aiplatform/model_monitoring/sampling.py index bff967772d..c2d067dc22 100644 --- a/google/cloud/aiplatform/model_monitoring/sampling.py +++ b/google/cloud/aiplatform/model_monitoring/sampling.py @@ -15,18 +15,13 @@ # limitations under the License. # -import abc from typing import Optional + from google.cloud.aiplatform_v1.types import ( model_monitoring as gca_model_monitoring, ) - -class _SamplingStrategy(abc.ABC): - """An abstract class for setting sampling strategy for model monitoring""" - - -class RandomSampleConfig(_SamplingStrategy): +class RandomSampleConfig: def __init__(self, sample_rate: Optional[float] = 1): """Initializer for RandomSampleConfig @@ -39,6 +34,7 @@ def __init__(self, sample_rate: Optional[float] = 1): self.sample_rate = sample_rate def as_proto(self): + """Returns RandomSampleConfig as a proto message""" return gca_model_monitoring.SamplingStrategy( random_sample_config=gca_model_monitoring.SamplingStrategy.RandomSampleConfig( sample_rate=self.sample_rate diff --git a/google/cloud/aiplatform/model_monitoring/schedule.py b/google/cloud/aiplatform/model_monitoring/schedule.py index 86610828e3..22eaca700b 100644 --- a/google/cloud/aiplatform/model_monitoring/schedule.py +++ b/google/cloud/aiplatform/model_monitoring/schedule.py @@ -15,18 +15,12 @@ # limitations under the License. # -import abc from google.protobuf import duration_pb2 # type: ignore from google.cloud.aiplatform_v1.types import ( model_deployment_monitoring_job as gca_model_deployment_monitoring_job, ) - -class _ScheduleConfig(abc.ABC): - """""" - - -class ScheduleConfig(_ScheduleConfig): +class ScheduleConfig: """""" def __init__(self, monitor_interval: int): @@ -41,6 +35,7 @@ def __init__(self, monitor_interval: int): self.monitor_interval = monitor_interval def as_proto(self): + """Returns ScheduleConfig as a proto message." return ( gca_model_deployment_monitoring_job.ModelDeploymentMonitoringScheduleConfig( monitor_interval=duration_pb2.Duration( From d9aaada772c5f60c162e91f0b1321c1a4dec6dd1 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Mon, 18 Jul 2022 16:33:09 -0700 Subject: [PATCH 43/53] adding more doc strings --- google/cloud/aiplatform/model_monitoring/objective.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/google/cloud/aiplatform/model_monitoring/objective.py b/google/cloud/aiplatform/model_monitoring/objective.py index a92b781c4d..3edf6a81dc 100644 --- a/google/cloud/aiplatform/model_monitoring/objective.py +++ b/google/cloud/aiplatform/model_monitoring/objective.py @@ -175,6 +175,15 @@ def __init__( ] = None, explanation_config: Optional["gca_model_monitoring._ExplanationConfig"] = None, ): + """Base class for EndpointObjectiveConfig. + Args: + skew_detection_config (_SkewDetectionConfig): + Optional. An instance of _SkewDetectionConfig. + drift_detection_config (_DriftDetectionConfig): + Optional. An instance of _DriftDetectionConfig. + explanation_config (_ExplanationConfig): + Optional. An instance of _ExplanationConfig. + """ self.skew_detection_config = skew_detection_config self.drift_detection_config = drift_detection_config self.explanation_config = explanation_config From 27d25f71f4a5911aca0363cf554454adf314766b Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Mon, 18 Jul 2022 16:54:39 -0700 Subject: [PATCH 44/53] more fixes --- google/cloud/aiplatform/jobs.py | 25 ++++++++++--------- .../aiplatform/model_monitoring/alert.py | 2 +- .../aiplatform/model_monitoring/objective.py | 8 +++--- .../aiplatform/model_monitoring/sampling.py | 5 ++-- .../aiplatform/model_monitoring/schedule.py | 6 ++--- .../unit/aiplatform/test_model_monitoring.py | 2 +- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 7fa0baa15b..2f70a674b4 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -2018,15 +2018,16 @@ def _parse_configs( all_configs = [] ## when same objective config is applied to SOME or ALL models - if deployed_model_ids is not None and not all( - model in all_models for model in deployed_model_ids - ): - error_string = ( - "Invalid model ID. The model ID must be one of [" - + ",".join(all_models) - + "]. Note that deployed model IDs are different from the uploaded model's ID" - ) - raise ValueError(error_string) + if deployed_model_ids is not None: + if not all(model in all_models for model in deployed_model_ids): + error_string = ( + "Invalid model ID. The model ID must be one of [" + + ",".join(all_models) + + "]. Note that deployed model IDs are different from the uploaded model's ID" + ) + raise ValueError(error_string) + else: + all_models = deployed_model_ids if isinstance(objective_configs, model_monitoring.EndpointObjectiveConfig): for model in all_models: @@ -2054,10 +2055,10 @@ def _parse_configs( + "]. Note that deployed model IDs are different from the uploaded model's ID" ) raise ValueError(error_string) - for key in objective_configs.keys(): + for (key, value) in objective_configs.items(): if ( model not in xai_enabled - and objective_configs[key].explanation_config is not None + and value.explanation_config is not None ): raise RuntimeError( "Invalid config for model ID %s. `explanation_config` should only be enabled if the model has `explanation_spec populated" @@ -2066,7 +2067,7 @@ def _parse_configs( all_configs.append( gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringObjectiveConfig( deployed_model_id=key, - objective_config=objective_configs[key].as_proto(), + objective_config=value.as_proto(), ) ) diff --git a/google/cloud/aiplatform/model_monitoring/alert.py b/google/cloud/aiplatform/model_monitoring/alert.py index 43e62afcfa..9eed27ec21 100644 --- a/google/cloud/aiplatform/model_monitoring/alert.py +++ b/google/cloud/aiplatform/model_monitoring/alert.py @@ -25,7 +25,7 @@ class EmailAlertConfig: def __init__( self, user_emails: List[str] = [], enable_logging: Optional[bool] = False ): - """Initializer for EmailAlertConfig + """Initializer for EmailAlertConfig. Args: user_emails (List[str]): diff --git a/google/cloud/aiplatform/model_monitoring/objective.py b/google/cloud/aiplatform/model_monitoring/objective.py index 3edf6a81dc..2c1b62f344 100644 --- a/google/cloud/aiplatform/model_monitoring/objective.py +++ b/google/cloud/aiplatform/model_monitoring/objective.py @@ -25,7 +25,7 @@ ) TF_RECORD = "tf-record" -CSV = "CSV" +CSV = "csv" JSONL = "jsonl" class _SkewDetectionConfig(abc.ABC): @@ -74,8 +74,6 @@ def __init__( "jsonl" The source file is a JSONL file. - - """ self.data_source = data_source self.skew_thresholds = skew_thresholds @@ -158,7 +156,7 @@ def __init__(self): self.enable_feature_attributes = False def as_proto(self): - """Returns _ExplanationConfig as a proto message""" + """Returns _ExplanationConfig as a proto message.""" return gca_model_monitoring.ModelMonitoringObjectiveConfig.ExplanationConfig( enable_feature_attributes=self.enable_feature_attributes ) @@ -310,7 +308,7 @@ def __init__( drift_thresholds: Optional[Dict[str, float]] = None, attribute_drift_thresholds: Optional[Dict[str, float]] = None, ): - """Initializer for EndpointDriftDetectionConfig + """Initializer for EndpointDriftDetectionConfig. Args: drift_thresholds (Dict[str, float]): diff --git a/google/cloud/aiplatform/model_monitoring/sampling.py b/google/cloud/aiplatform/model_monitoring/sampling.py index c2d067dc22..d056a63e6b 100644 --- a/google/cloud/aiplatform/model_monitoring/sampling.py +++ b/google/cloud/aiplatform/model_monitoring/sampling.py @@ -22,8 +22,9 @@ ) class RandomSampleConfig: + """A class that configures log sampling strategy.""" def __init__(self, sample_rate: Optional[float] = 1): - """Initializer for RandomSampleConfig + """Initializer for RandomSampleConfig. Args: sample_rate (float): @@ -34,7 +35,7 @@ def __init__(self, sample_rate: Optional[float] = 1): self.sample_rate = sample_rate def as_proto(self): - """Returns RandomSampleConfig as a proto message""" + """Returns RandomSampleConfig as a proto message.""" return gca_model_monitoring.SamplingStrategy( random_sample_config=gca_model_monitoring.SamplingStrategy.RandomSampleConfig( sample_rate=self.sample_rate diff --git a/google/cloud/aiplatform/model_monitoring/schedule.py b/google/cloud/aiplatform/model_monitoring/schedule.py index 22eaca700b..25db2a2f15 100644 --- a/google/cloud/aiplatform/model_monitoring/schedule.py +++ b/google/cloud/aiplatform/model_monitoring/schedule.py @@ -21,10 +21,10 @@ ) class ScheduleConfig: - """""" + """A class that configures model monitoring schedule.""" def __init__(self, monitor_interval: int): - """Initializer for ScheduleConfig + """Initializer for ScheduleConfig. Args: monitor_interval (int): @@ -35,7 +35,7 @@ def __init__(self, monitor_interval: int): self.monitor_interval = monitor_interval def as_proto(self): - """Returns ScheduleConfig as a proto message." + """Returns ScheduleConfig as a proto message.""" return ( gca_model_deployment_monitoring_job.ModelDeploymentMonitoringScheduleConfig( monitor_interval=duration_pb2.Duration( diff --git a/tests/unit/aiplatform/test_model_monitoring.py b/tests/unit/aiplatform/test_model_monitoring.py index e2504ad18e..20af3b9303 100644 --- a/tests/unit/aiplatform/test_model_monitoring.py +++ b/tests/unit/aiplatform/test_model_monitoring.py @@ -107,6 +107,6 @@ def test_invalid_data_format(self, data_source, data_format): data_format=data_format, ) assert ( - "Unsupported value. `data_format` must be one of 'tf-record', 'csv', or 'jsonl'" + "Unsupported value. `data_format` must be one of tf-record, csv, or jsonl" in str(e.value) ) From 274c69467c4fe57e2898b6aa48dc84890190e5d8 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Mon, 18 Jul 2022 23:56:01 +0000 Subject: [PATCH 45/53] formatting --- google/cloud/aiplatform/jobs.py | 7 ++---- .../aiplatform/model_monitoring/objective.py | 22 +++++++------------ .../aiplatform/model_monitoring/sampling.py | 2 ++ .../aiplatform/model_monitoring/schedule.py | 1 + 4 files changed, 13 insertions(+), 19 deletions(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 2f70a674b4..5e93cf0cd7 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -2056,10 +2056,7 @@ def _parse_configs( ) raise ValueError(error_string) for (key, value) in objective_configs.items(): - if ( - model not in xai_enabled - and value.explanation_config is not None - ): + if model not in xai_enabled and value.explanation_config is not None: raise RuntimeError( "Invalid config for model ID %s. `explanation_config` should only be enabled if the model has `explanation_spec populated" % model @@ -2108,7 +2105,7 @@ def create( Required. Endpoint resource name or an instance of `aiplatform.Endpoint`. Format: ``projects/{project}/locations/{location}/endpoints/{endpoint}`` - objective_configs (Union[model_monitoring.EndpointObjectiveConfig, + objective_configs (Union[model_monitoring.EndpointObjectiveConfig, Dict[str, model_monitoring.EndpointObjectiveConfig]]): Required. A single config if it applies to all models, or a dictionary of model_id: model_monitoring.objective.EndpointObjectiveConfig if diff --git a/google/cloud/aiplatform/model_monitoring/objective.py b/google/cloud/aiplatform/model_monitoring/objective.py index 2c1b62f344..d747c28b84 100644 --- a/google/cloud/aiplatform/model_monitoring/objective.py +++ b/google/cloud/aiplatform/model_monitoring/objective.py @@ -28,6 +28,7 @@ CSV = "csv" JSONL = "jsonl" + class _SkewDetectionConfig(abc.ABC): def __init__( self, @@ -112,7 +113,7 @@ def __init__( ): """Base class for prediction drift detection. Args: - drift_thresholds (Dict[str, float]): + drift_thresholds (Dict[str, float]): Required. Key is the feature name and value is the threshold. If a feature needs to be monitored for drift, a value threshold must be configured @@ -282,11 +283,10 @@ def __init__( ) elif data_source.startswith("gs:/"): training_dataset.gcs_source = gca_io.GcsSource(uris=[data_source]) - if data_format is not None and data_format not in [ - TF_RECORD, CSV, JSONL - ]: + if data_format is not None and data_format not in [TF_RECORD, CSV, JSONL]: raise ValueError( - "Unsupported value. `data_format` must be one of %s, %s, or %s" % (TF_RECORD, CSV, JSONL) + "Unsupported value. `data_format` must be one of %s, %s, or %s" + % (TF_RECORD, CSV, JSONL) ) training_dataset.data_format = data_format else: @@ -345,15 +345,9 @@ class EndpointObjectiveConfig(_ObjectiveConfig): def __init__( self, - skew_detection_config: Optional[ - "EndpointSkewDetectionConfig" - ] = None, - drift_detection_config: Optional[ - "EndpointDriftDetectionConfig" - ] = None, - explanation_config: Optional[ - "EndpointExplanationConfig" - ] = None, + skew_detection_config: Optional["EndpointSkewDetectionConfig"] = None, + drift_detection_config: Optional["EndpointDriftDetectionConfig"] = None, + explanation_config: Optional["EndpointExplanationConfig"] = None, ): """Initializer for EndpointObjectiveConfig. Args: diff --git a/google/cloud/aiplatform/model_monitoring/sampling.py b/google/cloud/aiplatform/model_monitoring/sampling.py index d056a63e6b..977d6b5162 100644 --- a/google/cloud/aiplatform/model_monitoring/sampling.py +++ b/google/cloud/aiplatform/model_monitoring/sampling.py @@ -21,8 +21,10 @@ model_monitoring as gca_model_monitoring, ) + class RandomSampleConfig: """A class that configures log sampling strategy.""" + def __init__(self, sample_rate: Optional[float] = 1): """Initializer for RandomSampleConfig. diff --git a/google/cloud/aiplatform/model_monitoring/schedule.py b/google/cloud/aiplatform/model_monitoring/schedule.py index 25db2a2f15..737d6df077 100644 --- a/google/cloud/aiplatform/model_monitoring/schedule.py +++ b/google/cloud/aiplatform/model_monitoring/schedule.py @@ -20,6 +20,7 @@ model_deployment_monitoring_job as gca_model_deployment_monitoring_job, ) + class ScheduleConfig: """A class that configures model monitoring schedule.""" From cd997fbcb3bc7e46d7e8c2e7f39f89193777ab11 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Mon, 18 Jul 2022 23:58:01 +0000 Subject: [PATCH 46/53] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- google/cloud/aiplatform/jobs.py | 7 ++---- .../aiplatform/model_monitoring/objective.py | 22 +++++++------------ .../aiplatform/model_monitoring/sampling.py | 2 ++ .../aiplatform/model_monitoring/schedule.py | 1 + 4 files changed, 13 insertions(+), 19 deletions(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 2f70a674b4..5e93cf0cd7 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -2056,10 +2056,7 @@ def _parse_configs( ) raise ValueError(error_string) for (key, value) in objective_configs.items(): - if ( - model not in xai_enabled - and value.explanation_config is not None - ): + if model not in xai_enabled and value.explanation_config is not None: raise RuntimeError( "Invalid config for model ID %s. `explanation_config` should only be enabled if the model has `explanation_spec populated" % model @@ -2108,7 +2105,7 @@ def create( Required. Endpoint resource name or an instance of `aiplatform.Endpoint`. Format: ``projects/{project}/locations/{location}/endpoints/{endpoint}`` - objective_configs (Union[model_monitoring.EndpointObjectiveConfig, + objective_configs (Union[model_monitoring.EndpointObjectiveConfig, Dict[str, model_monitoring.EndpointObjectiveConfig]]): Required. A single config if it applies to all models, or a dictionary of model_id: model_monitoring.objective.EndpointObjectiveConfig if diff --git a/google/cloud/aiplatform/model_monitoring/objective.py b/google/cloud/aiplatform/model_monitoring/objective.py index 2c1b62f344..d747c28b84 100644 --- a/google/cloud/aiplatform/model_monitoring/objective.py +++ b/google/cloud/aiplatform/model_monitoring/objective.py @@ -28,6 +28,7 @@ CSV = "csv" JSONL = "jsonl" + class _SkewDetectionConfig(abc.ABC): def __init__( self, @@ -112,7 +113,7 @@ def __init__( ): """Base class for prediction drift detection. Args: - drift_thresholds (Dict[str, float]): + drift_thresholds (Dict[str, float]): Required. Key is the feature name and value is the threshold. If a feature needs to be monitored for drift, a value threshold must be configured @@ -282,11 +283,10 @@ def __init__( ) elif data_source.startswith("gs:/"): training_dataset.gcs_source = gca_io.GcsSource(uris=[data_source]) - if data_format is not None and data_format not in [ - TF_RECORD, CSV, JSONL - ]: + if data_format is not None and data_format not in [TF_RECORD, CSV, JSONL]: raise ValueError( - "Unsupported value. `data_format` must be one of %s, %s, or %s" % (TF_RECORD, CSV, JSONL) + "Unsupported value. `data_format` must be one of %s, %s, or %s" + % (TF_RECORD, CSV, JSONL) ) training_dataset.data_format = data_format else: @@ -345,15 +345,9 @@ class EndpointObjectiveConfig(_ObjectiveConfig): def __init__( self, - skew_detection_config: Optional[ - "EndpointSkewDetectionConfig" - ] = None, - drift_detection_config: Optional[ - "EndpointDriftDetectionConfig" - ] = None, - explanation_config: Optional[ - "EndpointExplanationConfig" - ] = None, + skew_detection_config: Optional["EndpointSkewDetectionConfig"] = None, + drift_detection_config: Optional["EndpointDriftDetectionConfig"] = None, + explanation_config: Optional["EndpointExplanationConfig"] = None, ): """Initializer for EndpointObjectiveConfig. Args: diff --git a/google/cloud/aiplatform/model_monitoring/sampling.py b/google/cloud/aiplatform/model_monitoring/sampling.py index d056a63e6b..977d6b5162 100644 --- a/google/cloud/aiplatform/model_monitoring/sampling.py +++ b/google/cloud/aiplatform/model_monitoring/sampling.py @@ -21,8 +21,10 @@ model_monitoring as gca_model_monitoring, ) + class RandomSampleConfig: """A class that configures log sampling strategy.""" + def __init__(self, sample_rate: Optional[float] = 1): """Initializer for RandomSampleConfig. diff --git a/google/cloud/aiplatform/model_monitoring/schedule.py b/google/cloud/aiplatform/model_monitoring/schedule.py index 25db2a2f15..737d6df077 100644 --- a/google/cloud/aiplatform/model_monitoring/schedule.py +++ b/google/cloud/aiplatform/model_monitoring/schedule.py @@ -20,6 +20,7 @@ model_deployment_monitoring_job as gca_model_deployment_monitoring_job, ) + class ScheduleConfig: """A class that configures model monitoring schedule.""" From 4640189c2d12bdbca18957cb52a846b30a3306fd Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Tue, 19 Jul 2022 17:03:13 -0700 Subject: [PATCH 47/53] added more test coverage and changed iterator names for parse_configs function --- google/cloud/aiplatform/jobs.py | 10 +++--- .../aiplatform/test_model_monitoring.py | 34 +++++++++++++++++++ 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 5e93cf0cd7..a6f0162822 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -2055,16 +2055,16 @@ def _parse_configs( + "]. Note that deployed model IDs are different from the uploaded model's ID" ) raise ValueError(error_string) - for (key, value) in objective_configs.items(): - if model not in xai_enabled and value.explanation_config is not None: + for (deployed_model, objective_config) in objective_configs.items(): + if deployed_model not in xai_enabled and objective_config.explanation_config is not None: raise RuntimeError( "Invalid config for model ID %s. `explanation_config` should only be enabled if the model has `explanation_spec populated" - % model + % deployed_model ) all_configs.append( gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringObjectiveConfig( - deployed_model_id=key, - objective_config=value.as_proto(), + deployed_model_id=deployed_model, + objective_config=objective_config.as_proto(), ) ) diff --git a/tests/system/aiplatform/test_model_monitoring.py b/tests/system/aiplatform/test_model_monitoring.py index 3551a7641c..fcb6dec22a 100644 --- a/tests/system/aiplatform/test_model_monitoring.py +++ b/tests/system/aiplatform/test_model_monitoring.py @@ -362,3 +362,37 @@ def test_mdm_invalid_config_xai(self, shared_state): "`explanation_config` should only be enabled if the model has `explanation_spec populated" in str(e.value) ) + + def test_mdm_two_models_invalid_configs_xai(self, shared_state): + temp_endpoint_with_two_models = self.temp_endpoint_with_two_models(shared_state) + [deployed_model1, deployed_model2] = list( + map(lambda x: x.id, temp_endpoint_with_two_models.list_models()) + ) + objective_config.explanation_config = ( + model_monitoring.EndpointExplanationConfig() + ) + all_configs = { + deployed_model1: objective_config, + deployed_model2: objective_config2, + } + with pytest.raises(RuntimeError) as e: + objective_config.explanation_config = ( + model_monitoring.EndpointExplanationConfig() + ) + aiplatform.ModelDeploymentMonitoringJob.create( + display_name=JOB_NAME, + logging_sampling_strategy=sampling_strategy, + schedule_config=schedule_config, + alert_config=alert_config, + objective_configs=objective_config, + create_request_timeout=3600, + project=e2e_base._PROJECT, + location=e2e_base._LOCATION, + endpoint=temp_endpoint_with_two_models, + predict_instance_schema_uri="", + analysis_instance_schema_uri="", + ) + assert ( + "`explanation_config` should only be enabled if the model has `explanation_spec populated" + in str(e.value) + ) From bbadaa13e1f7e542bc4e2d8d5efb820c2a5fd544 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Wed, 20 Jul 2022 00:07:04 +0000 Subject: [PATCH 48/53] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- google/cloud/aiplatform/jobs.py | 5 ++++- tests/system/aiplatform/test_model_monitoring.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index a6f0162822..a4b0e2c81a 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -2056,7 +2056,10 @@ def _parse_configs( ) raise ValueError(error_string) for (deployed_model, objective_config) in objective_configs.items(): - if deployed_model not in xai_enabled and objective_config.explanation_config is not None: + if ( + deployed_model not in xai_enabled + and objective_config.explanation_config is not None + ): raise RuntimeError( "Invalid config for model ID %s. `explanation_config` should only be enabled if the model has `explanation_spec populated" % deployed_model diff --git a/tests/system/aiplatform/test_model_monitoring.py b/tests/system/aiplatform/test_model_monitoring.py index fcb6dec22a..e587833f2f 100644 --- a/tests/system/aiplatform/test_model_monitoring.py +++ b/tests/system/aiplatform/test_model_monitoring.py @@ -369,7 +369,7 @@ def test_mdm_two_models_invalid_configs_xai(self, shared_state): map(lambda x: x.id, temp_endpoint_with_two_models.list_models()) ) objective_config.explanation_config = ( - model_monitoring.EndpointExplanationConfig() + model_monitoring.EndpointExplanationConfig() ) all_configs = { deployed_model1: objective_config, From 0d8a045ff291133545ae7ba328130792ed22e6c3 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Tue, 19 Jul 2022 17:07:39 -0700 Subject: [PATCH 49/53] making objective config class non-abstract --- google/cloud/aiplatform/model_monitoring/objective.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/google/cloud/aiplatform/model_monitoring/objective.py b/google/cloud/aiplatform/model_monitoring/objective.py index d747c28b84..6c158037d6 100644 --- a/google/cloud/aiplatform/model_monitoring/objective.py +++ b/google/cloud/aiplatform/model_monitoring/objective.py @@ -15,7 +15,6 @@ # limitations under the License. # -import abc from typing import Optional, Dict from google.cloud.aiplatform_v1.types import ( @@ -29,7 +28,7 @@ JSONL = "jsonl" -class _SkewDetectionConfig(abc.ABC): +class _SkewDetectionConfig: def __init__( self, data_source: str, @@ -105,7 +104,7 @@ def as_proto(self): ) -class _DriftDetectionConfig(abc.ABC): +class _DriftDetectionConfig: def __init__( self, drift_thresholds: Dict[str, float], @@ -151,7 +150,7 @@ def as_proto(self): ) -class _ExplanationConfig(abc.ABC): +class _ExplanationConfig: def __init__(self): """Base class for EndpointExplanationConfig.""" self.enable_feature_attributes = False @@ -163,7 +162,7 @@ def as_proto(self): ) -class _ObjectiveConfig(abc.ABC): +class _ObjectiveConfig: def __init__( self, skew_detection_config: Optional[ From 2723da744ec31572cf7fc66fcc825851530757c1 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Wed, 27 Jul 2022 15:56:33 -0700 Subject: [PATCH 50/53] renaming configuration classes --- google/cloud/aiplatform/__init__.py | 8 ++-- google/cloud/aiplatform/jobs.py | 34 ++++++++-------- .../aiplatform/model_monitoring/__init__.py | 16 ++++---- .../aiplatform/model_monitoring/objective.py | 40 +++++++++---------- .../aiplatform/test_model_monitoring.py | 20 +++++----- .../unit/aiplatform/test_model_monitoring.py | 10 ++--- 6 files changed, 64 insertions(+), 64 deletions(-) diff --git a/google/cloud/aiplatform/__init__.py b/google/cloud/aiplatform/__init__.py index a3b49b9ea3..8107756229 100644 --- a/google/cloud/aiplatform/__init__.py +++ b/google/cloud/aiplatform/__init__.py @@ -128,10 +128,10 @@ "CustomPythonPackageTrainingJob", "EmailAlertConfig", "Endpoint", - "EndpointDriftDetectionConfig", - "EndpointExplanationConfig", - "EndpointObjectiveConfig", - "EndpointSkewDetectionConfig", + "DriftDetectionConfig", + "ExplanationConfig", + "ObjectiveConfig", + "SkewDetectionConfig", "EntityType", "Execution", "Experiment", diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index a4b0e2c81a..8787b6f1f0 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -1974,8 +1974,8 @@ def __init__( def _parse_configs( cls, objective_configs: Union[ - model_monitoring.EndpointObjectiveConfig, - Dict[str, model_monitoring.EndpointObjectiveConfig], + model_monitoring.ObjectiveConfig, + Dict[str, model_monitoring.ObjectiveConfig], ], endpoint: "aiplatform.Endpoint", deployed_model_ids: Optional[List[str]] = None, @@ -1985,10 +1985,10 @@ def _parse_configs( """Helper function for matching objective configs with their corresponding models. Args: - objective_configs (Union[model_monitoring.objective.EndpointObjectiveConfig, - Dict[str, model_monitoring.objective.EndpointObjectiveConfig]): + objective_configs (Union[model_monitoring.objective.ObjectiveConfig, + Dict[str, model_monitoring.objective.ObjectiveConfig]): Required. A single config if it applies to all models, or a dictionary of - model_id: model_monitoring.objective.EndpointObjectiveConfig if + model_id: model_monitoring.objective.ObjectiveConfig if different model IDs have different configs. endpoint (aiplatform.Endpoint): Required. A valid instance of aiplatforn.Endpoint to launch the MDM job on. @@ -1998,7 +1998,7 @@ def _parse_configs( uploaded model ID, and IDs in this list should consist of deployed model IDs on the same endpoint passed in the argument. If `objective_configs` is a dictionary, then this parameter is ignored. If `objective_configs` is an instance of - `model_monitoring.EndpointObjectiveConfig` and `deployed_model_ids` is a non-empty + `model_monitoring.ObjectiveConfig` and `deployed_model_ids` is a non-empty list of valid IDs, then the same objective config will apply to all models in this list. Returns: @@ -2029,7 +2029,7 @@ def _parse_configs( else: all_models = deployed_model_ids - if isinstance(objective_configs, model_monitoring.EndpointObjectiveConfig): + if isinstance(objective_configs, model_monitoring.ObjectiveConfig): for model in all_models: if ( model not in xai_enabled @@ -2079,8 +2079,8 @@ def create( endpoint: Union[str, "aiplatform.Endpoint"], objective_configs: Optional[ Union[ - model_monitoring.EndpointObjectiveConfig, - Dict[str, model_monitoring.EndpointObjectiveConfig], + model_monitoring.ObjectiveConfig, + Dict[str, model_monitoring.ObjectiveConfig], ] ] = None, logging_sampling_strategy: Optional[model_monitoring.RandomSampleConfig] = None, @@ -2108,10 +2108,10 @@ def create( Required. Endpoint resource name or an instance of `aiplatform.Endpoint`. Format: ``projects/{project}/locations/{location}/endpoints/{endpoint}`` - objective_configs (Union[model_monitoring.EndpointObjectiveConfig, - Dict[str, model_monitoring.EndpointObjectiveConfig]]): + objective_configs (Union[model_monitoring.ObjectiveConfig, + Dict[str, model_monitoring.ObjectiveConfig]]): Required. A single config if it applies to all models, or a dictionary of - model_id: model_monitoring.objective.EndpointObjectiveConfig if + model_id: model_monitoring.objective.ObjectiveConfig if different model IDs have different configs. logging_sampling_strategy (model_monitoring.sampling.RandomSampleConfig): @@ -2297,8 +2297,8 @@ def update( enable_monitoring_pipeline_logs: Optional[bool] = None, objective_configs: Optional[ Union[ - model_monitoring.EndpointObjectiveConfig, - Dict[str, model_monitoring.EndpointObjectiveConfig], + model_monitoring.ObjectiveConfig, + Dict[str, model_monitoring.ObjectiveConfig], ] ] = None, deployed_model_ids: Optional[List[str]] = None, @@ -2347,10 +2347,10 @@ def update( pricing `__. objective_configs (Union[ - Required. model_monitoring.objective.EndpointObjectiveConfig, - Dict[str, model_monitoring.objective.EndpointObjectiveConfig]): + Required. model_monitoring.objective.ObjectiveConfig, + Dict[str, model_monitoring.objective.ObjectiveConfig]): A single config if it applies to all models, or a dictionary of - model_id: model_monitoring.objective.EndpointObjectiveConfig if + model_id: model_monitoring.objective.ObjectiveConfig if different model IDs have different configs. deployed_model_ids (List[str]): diff --git a/google/cloud/aiplatform/model_monitoring/__init__.py b/google/cloud/aiplatform/model_monitoring/__init__.py index 2935ff12cb..c4562ff147 100644 --- a/google/cloud/aiplatform/model_monitoring/__init__.py +++ b/google/cloud/aiplatform/model_monitoring/__init__.py @@ -17,20 +17,20 @@ from google.cloud.aiplatform.model_monitoring.alert import EmailAlertConfig from google.cloud.aiplatform.model_monitoring.objective import ( - EndpointSkewDetectionConfig, - EndpointDriftDetectionConfig, - EndpointExplanationConfig, - EndpointObjectiveConfig, + SkewDetectionConfig, + DriftDetectionConfig, + ExplanationConfig, + ObjectiveConfig, ) from google.cloud.aiplatform.model_monitoring.sampling import RandomSampleConfig from google.cloud.aiplatform.model_monitoring.schedule import ScheduleConfig __all__ = ( "EmailAlertConfig", - "EndpointSkewDetectionConfig", - "EndpointDriftDetectionConfig", - "EndpointExplanationConfig", - "EndpointObjectiveConfig", + "SkewDetectionConfig", + "DriftDetectionConfig", + "ExplanationConfig", + "ObjectiveConfig", "RandomSampleConfig", "ScheduleConfig", ) diff --git a/google/cloud/aiplatform/model_monitoring/objective.py b/google/cloud/aiplatform/model_monitoring/objective.py index 6c158037d6..a7800a3485 100644 --- a/google/cloud/aiplatform/model_monitoring/objective.py +++ b/google/cloud/aiplatform/model_monitoring/objective.py @@ -152,7 +152,7 @@ def as_proto(self): class _ExplanationConfig: def __init__(self): - """Base class for EndpointExplanationConfig.""" + """Base class for ExplanationConfig.""" self.enable_feature_attributes = False def as_proto(self): @@ -173,7 +173,7 @@ def __init__( ] = None, explanation_config: Optional["gca_model_monitoring._ExplanationConfig"] = None, ): - """Base class for EndpointObjectiveConfig. + """Base class for ObjectiveConfig. Args: skew_detection_config (_SkewDetectionConfig): Optional. An instance of _SkewDetectionConfig. @@ -205,7 +205,7 @@ def as_proto(self): ) -class EndpointSkewDetectionConfig(_SkewDetectionConfig): +class SkewDetectionConfig(_SkewDetectionConfig): """A class that configures skew detection for models deployed to an endpoint. Training-serving skew occurs when input data in production has a different @@ -221,7 +221,7 @@ def __init__( attribute_skew_thresholds: Optional[Dict[str, float]] = None, data_format: Optional[str] = None, ): - """Initializer for EndpointSkewDetectionConfig. + """Initializer for SkewDetectionConfig. Args: data_source (str): @@ -293,12 +293,12 @@ def __init__( self.training_dataset = training_dataset -class EndpointDriftDetectionConfig(_DriftDetectionConfig): +class DriftDetectionConfig(_DriftDetectionConfig): """A class that configures prediction drift detection for models deployed to an endpoint. Prediction drift occurs when feature data distribution changes noticeably over time, and should be set when the original training data is unavailable. - If original training data is available, EndpointSkewDetectionConfig should + If original training data is available, SkewDetectionConfig should be set instead. """ @@ -307,7 +307,7 @@ def __init__( drift_thresholds: Optional[Dict[str, float]] = None, attribute_drift_thresholds: Optional[Dict[str, float]] = None, ): - """Initializer for EndpointDriftDetectionConfig. + """Initializer for DriftDetectionConfig. Args: drift_thresholds (Dict[str, float]): @@ -327,35 +327,35 @@ def __init__( super().__init__(drift_thresholds, attribute_drift_thresholds) -class EndpointExplanationConfig(_ExplanationConfig): +class ExplanationConfig(_ExplanationConfig): """A class that enables Vertex Explainable AI. Only applicable if the model has explanation_spec populated. By default, explanation config is disabled. Instantiating this class will enable the config. """ def __init__(self): - """Initializer for EndpointExplanationConfig.""" + """Initializer for ExplanationConfig.""" super().__init__() self.enable_feature_attributes = True -class EndpointObjectiveConfig(_ObjectiveConfig): +class ObjectiveConfig(_ObjectiveConfig): """A class that captures skew detection, drift detection, and explanation configs.""" def __init__( self, - skew_detection_config: Optional["EndpointSkewDetectionConfig"] = None, - drift_detection_config: Optional["EndpointDriftDetectionConfig"] = None, - explanation_config: Optional["EndpointExplanationConfig"] = None, + skew_detection_config: Optional["SkewDetectionConfig"] = None, + drift_detection_config: Optional["DriftDetectionConfig"] = None, + explanation_config: Optional["ExplanationConfig"] = None, ): - """Initializer for EndpointObjectiveConfig. + """Initializer for ObjectiveConfig. Args: - skew_detection_config (EndpointSkewDetectionConfig): - Optional. An instance of EndpointSkewDetectionConfig. - drift_detection_config (EndpointDriftDetectionConfig): - Optional. An instance of EndpointDriftDetectionConfig. - explanation_config (EndpointExplanationConfig): - Optional. An instance of EndpointExplanationConfig. + skew_detection_config (SkewDetectionConfig): + Optional. An instance of SkewDetectionConfig. + drift_detection_config (DriftDetectionConfig): + Optional. An instance of DriftDetectionConfig. + explanation_config (ExplanationConfig): + Optional. An instance of ExplanationConfig. """ super().__init__( skew_detection_config, drift_detection_config, explanation_config diff --git a/tests/system/aiplatform/test_model_monitoring.py b/tests/system/aiplatform/test_model_monitoring.py index e587833f2f..cc264c699c 100644 --- a/tests/system/aiplatform/test_model_monitoring.py +++ b/tests/system/aiplatform/test_model_monitoring.py @@ -106,26 +106,26 @@ schedule_config = model_monitoring.ScheduleConfig(monitor_interval=MONITOR_INTERVAL) -skew_config = model_monitoring.EndpointSkewDetectionConfig( +skew_config = model_monitoring.SkewDetectionConfig( data_source=DATASET_BQ_URI, skew_thresholds=skew_thresholds, attribute_skew_thresholds=attrib_skew_thresholds, target_field=TARGET, ) -drift_config = model_monitoring.EndpointDriftDetectionConfig( +drift_config = model_monitoring.DriftDetectionConfig( drift_thresholds=drift_thresholds, attribute_drift_thresholds=attrib_drift_thresholds, ) -drift_config2 = model_monitoring.EndpointDriftDetectionConfig( +drift_config2 = model_monitoring.DriftDetectionConfig( drift_thresholds=drift_thresholds, attribute_drift_thresholds=ATTRIB_DRIFT_DEFAULT_THRESHOLDS, ) -objective_config = model_monitoring.EndpointObjectiveConfig(skew_config, drift_config) +objective_config = model_monitoring.ObjectiveConfig(skew_config, drift_config) -objective_config2 = model_monitoring.EndpointObjectiveConfig(skew_config, drift_config2) +objective_config2 = model_monitoring.ObjectiveConfig(skew_config, drift_config2) @pytest.mark.usefixtures("tear_down_resources") @@ -236,7 +236,7 @@ def test_mdm_one_model_one_valid_config(self, shared_state): # test job update, pause, resume, and delete() timeout = time.time() + 3600 - new_obj_config = model_monitoring.EndpointObjectiveConfig(skew_config) + new_obj_config = model_monitoring.ObjectiveConfig(skew_config) while time.time() < timeout: if job.state == gca_job_state.JobState.JOB_STATE_RUNNING: @@ -343,7 +343,7 @@ def test_mdm_invalid_config_xai(self, shared_state): temp_endpoint = self.temp_endpoint(shared_state) with pytest.raises(RuntimeError) as e: objective_config.explanation_config = ( - model_monitoring.EndpointExplanationConfig() + model_monitoring.ExplanationConfig() ) aiplatform.ModelDeploymentMonitoringJob.create( display_name=JOB_NAME, @@ -369,7 +369,7 @@ def test_mdm_two_models_invalid_configs_xai(self, shared_state): map(lambda x: x.id, temp_endpoint_with_two_models.list_models()) ) objective_config.explanation_config = ( - model_monitoring.EndpointExplanationConfig() + model_monitoring.ExplanationConfig() ) all_configs = { deployed_model1: objective_config, @@ -377,14 +377,14 @@ def test_mdm_two_models_invalid_configs_xai(self, shared_state): } with pytest.raises(RuntimeError) as e: objective_config.explanation_config = ( - model_monitoring.EndpointExplanationConfig() + model_monitoring.ExplanationConfig() ) aiplatform.ModelDeploymentMonitoringJob.create( display_name=JOB_NAME, logging_sampling_strategy=sampling_strategy, schedule_config=schedule_config, alert_config=alert_config, - objective_configs=objective_config, + objective_configs=all_configs, create_request_timeout=3600, project=e2e_base._PROJECT, location=e2e_base._LOCATION, diff --git a/tests/unit/aiplatform/test_model_monitoring.py b/tests/unit/aiplatform/test_model_monitoring.py index 20af3b9303..a29aa060db 100644 --- a/tests/unit/aiplatform/test_model_monitoring.py +++ b/tests/unit/aiplatform/test_model_monitoring.py @@ -51,11 +51,11 @@ def test_valid_configs(self, data_source, data_format): user_emails=[_TEST_EMAIL1, _TEST_EMAIL2] ) - prediction_drift_config = model_monitoring.EndpointDriftDetectionConfig( + prediction_drift_config = model_monitoring.DriftDetectionConfig( drift_thresholds={_TEST_KEY: _TEST_THRESHOLD} ) - skew_config = model_monitoring.EndpointSkewDetectionConfig( + skew_config = model_monitoring.SkewDetectionConfig( data_source=data_source, skew_thresholds={_TEST_KEY: _TEST_THRESHOLD}, target_field=_TEST_TARGET_FIELD, @@ -63,9 +63,9 @@ def test_valid_configs(self, data_source, data_format): data_format=data_format, ) - xai_config = model_monitoring.EndpointExplanationConfig() + xai_config = model_monitoring.ExplanationConfig() - objective_config = model_monitoring.EndpointObjectiveConfig( + objective_config = model_monitoring.ObjectiveConfig( skew_detection_config=skew_config, drift_detection_config=prediction_drift_config, explanation_config=xai_config, @@ -99,7 +99,7 @@ def test_valid_configs(self, data_source, data_format): def test_invalid_data_format(self, data_source, data_format): if data_format == "other": with pytest.raises(ValueError) as e: - model_monitoring.EndpointSkewDetectionConfig( + model_monitoring.SkewDetectionConfig( data_source=data_source, skew_thresholds={_TEST_KEY: _TEST_THRESHOLD}, target_field=_TEST_TARGET_FIELD, From 481227b68cb6839d7d33d23e71878c64950899dd Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Wed, 27 Jul 2022 22:59:42 +0000 Subject: [PATCH 51/53] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- tests/system/aiplatform/test_model_monitoring.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/system/aiplatform/test_model_monitoring.py b/tests/system/aiplatform/test_model_monitoring.py index cc264c699c..d066bdae62 100644 --- a/tests/system/aiplatform/test_model_monitoring.py +++ b/tests/system/aiplatform/test_model_monitoring.py @@ -342,9 +342,7 @@ def test_mdm_invalid_config_incorrect_model_id(self, shared_state): def test_mdm_invalid_config_xai(self, shared_state): temp_endpoint = self.temp_endpoint(shared_state) with pytest.raises(RuntimeError) as e: - objective_config.explanation_config = ( - model_monitoring.ExplanationConfig() - ) + objective_config.explanation_config = model_monitoring.ExplanationConfig() aiplatform.ModelDeploymentMonitoringJob.create( display_name=JOB_NAME, logging_sampling_strategy=sampling_strategy, @@ -368,17 +366,13 @@ def test_mdm_two_models_invalid_configs_xai(self, shared_state): [deployed_model1, deployed_model2] = list( map(lambda x: x.id, temp_endpoint_with_two_models.list_models()) ) - objective_config.explanation_config = ( - model_monitoring.ExplanationConfig() - ) + objective_config.explanation_config = model_monitoring.ExplanationConfig() all_configs = { deployed_model1: objective_config, deployed_model2: objective_config2, } with pytest.raises(RuntimeError) as e: - objective_config.explanation_config = ( - model_monitoring.ExplanationConfig() - ) + objective_config.explanation_config = model_monitoring.ExplanationConfig() aiplatform.ModelDeploymentMonitoringJob.create( display_name=JOB_NAME, logging_sampling_strategy=sampling_strategy, From d6b04a22d90c33bafbb9dfa8328972308ae7a8d7 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Wed, 27 Jul 2022 23:00:58 +0000 Subject: [PATCH 52/53] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- tests/system/aiplatform/test_model_monitoring.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/system/aiplatform/test_model_monitoring.py b/tests/system/aiplatform/test_model_monitoring.py index cc264c699c..d066bdae62 100644 --- a/tests/system/aiplatform/test_model_monitoring.py +++ b/tests/system/aiplatform/test_model_monitoring.py @@ -342,9 +342,7 @@ def test_mdm_invalid_config_incorrect_model_id(self, shared_state): def test_mdm_invalid_config_xai(self, shared_state): temp_endpoint = self.temp_endpoint(shared_state) with pytest.raises(RuntimeError) as e: - objective_config.explanation_config = ( - model_monitoring.ExplanationConfig() - ) + objective_config.explanation_config = model_monitoring.ExplanationConfig() aiplatform.ModelDeploymentMonitoringJob.create( display_name=JOB_NAME, logging_sampling_strategy=sampling_strategy, @@ -368,17 +366,13 @@ def test_mdm_two_models_invalid_configs_xai(self, shared_state): [deployed_model1, deployed_model2] = list( map(lambda x: x.id, temp_endpoint_with_two_models.list_models()) ) - objective_config.explanation_config = ( - model_monitoring.ExplanationConfig() - ) + objective_config.explanation_config = model_monitoring.ExplanationConfig() all_configs = { deployed_model1: objective_config, deployed_model2: objective_config2, } with pytest.raises(RuntimeError) as e: - objective_config.explanation_config = ( - model_monitoring.ExplanationConfig() - ) + objective_config.explanation_config = model_monitoring.ExplanationConfig() aiplatform.ModelDeploymentMonitoringJob.create( display_name=JOB_NAME, logging_sampling_strategy=sampling_strategy, From 6a4b062319657705b1bb8c7d5270ac1c77dd7f00 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Wed, 27 Jul 2022 23:01:45 +0000 Subject: [PATCH 53/53] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- tests/system/aiplatform/test_model_monitoring.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/system/aiplatform/test_model_monitoring.py b/tests/system/aiplatform/test_model_monitoring.py index cc264c699c..d066bdae62 100644 --- a/tests/system/aiplatform/test_model_monitoring.py +++ b/tests/system/aiplatform/test_model_monitoring.py @@ -342,9 +342,7 @@ def test_mdm_invalid_config_incorrect_model_id(self, shared_state): def test_mdm_invalid_config_xai(self, shared_state): temp_endpoint = self.temp_endpoint(shared_state) with pytest.raises(RuntimeError) as e: - objective_config.explanation_config = ( - model_monitoring.ExplanationConfig() - ) + objective_config.explanation_config = model_monitoring.ExplanationConfig() aiplatform.ModelDeploymentMonitoringJob.create( display_name=JOB_NAME, logging_sampling_strategy=sampling_strategy, @@ -368,17 +366,13 @@ def test_mdm_two_models_invalid_configs_xai(self, shared_state): [deployed_model1, deployed_model2] = list( map(lambda x: x.id, temp_endpoint_with_two_models.list_models()) ) - objective_config.explanation_config = ( - model_monitoring.ExplanationConfig() - ) + objective_config.explanation_config = model_monitoring.ExplanationConfig() all_configs = { deployed_model1: objective_config, deployed_model2: objective_config2, } with pytest.raises(RuntimeError) as e: - objective_config.explanation_config = ( - model_monitoring.ExplanationConfig() - ) + objective_config.explanation_config = model_monitoring.ExplanationConfig() aiplatform.ModelDeploymentMonitoringJob.create( display_name=JOB_NAME, logging_sampling_strategy=sampling_strategy,