diff --git a/google/oauth2/_client.py b/google/oauth2/_client.py
index 448716329..9819d4a8d 100644
--- a/google/oauth2/_client.py
+++ b/google/oauth2/_client.py
@@ -108,7 +108,13 @@ def _token_endpoint_request(request, token_uri, body):
             if hasattr(response.data, "decode")
             else response.data
         )
-        response_data = json.loads(response_body)
+        try:
+            response_data = json.loads(response_body)
+        except (KeyError, ValueError):
+            # raise if the response can't be json decoded.
+            error_details = response_body
+            raise exceptions.RefreshError(error_details, response_body)
+
 
         if response.status == http_client.OK:
             break