From 899b82dcb9a024eb3adf6223ef5f7cd590ac644b Mon Sep 17 00:00:00 2001 From: Tugrulhan Karsli Date: Wed, 12 Aug 2026 12:10:22 +0300 Subject: [PATCH] feat(azuredevops): add Azure DevOps Server (On-Premises) support (#9014) --- .../azuredevops_go/api/azuredevops/client.go | 48 ++++-- .../api/azuredevops/client_onprem_test.go | 107 +++++++++++++ .../azuredevops_go/api/blueprint_v200.go | 6 +- .../azuredevops_go/api/connection_api.go | 49 ++++-- .../azuredevops_go/api/remote_helper.go | 46 ++++-- .../azuredevops_go/models/connection.go | 38 ++++- .../azuredevops_go/models/connection_test.go | 144 ++++++++++++++++++ .../20260722_add_endpoint_to_azuredevops.go | 49 ++++++ .../20260726_add_username_to_azuredevops.go | 49 ++++++ .../migrationscripts/archived/connection.go | 1 + .../models/migrationscripts/register.go | 2 + .../azuredevops_go/tasks/account_collector.go | 5 + .../tasks/ci_cd_build_collector.go | 2 +- .../tasks/ci_cd_timeline_records_collector.go | 2 +- .../azuredevops_go/tasks/commit_collector.go | 2 +- .../azuredevops_go/tasks/pr_collector.go | 2 +- .../tasks/pr_commit_collector.go | 2 +- .../gitextractor/parser/clone_gitcli.go | 39 +++-- .../data-scope-remote/search-local.tsx | 2 +- .../data-scope-remote/search-remote.tsx | 4 +- .../components/data-scope-select/index.tsx | 6 +- .../src/plugins/register/azure/config.tsx | 20 ++- .../azure/connection-fields/base-url.tsx | 47 +++++- .../azure/connection-fields/organization.tsx | 8 +- 24 files changed, 596 insertions(+), 84 deletions(-) create mode 100644 backend/plugins/azuredevops_go/api/azuredevops/client_onprem_test.go create mode 100644 backend/plugins/azuredevops_go/models/connection_test.go create mode 100644 backend/plugins/azuredevops_go/models/migrationscripts/20260722_add_endpoint_to_azuredevops.go create mode 100644 backend/plugins/azuredevops_go/models/migrationscripts/20260726_add_username_to_azuredevops.go diff --git a/backend/plugins/azuredevops_go/api/azuredevops/client.go b/backend/plugins/azuredevops_go/api/azuredevops/client.go index bfe4beb6d0e..207967f438e 100644 --- a/backend/plugins/azuredevops_go/api/azuredevops/client.go +++ b/backend/plugins/azuredevops_go/api/azuredevops/client.go @@ -20,15 +20,17 @@ package azuredevops import ( "encoding/json" "fmt" - "github.com/apache/incubator-devlake/core/errors" - "github.com/apache/incubator-devlake/core/plugin" - "github.com/apache/incubator-devlake/helpers/pluginhelper/api" - "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" "io" "net/http" "net/url" "strconv" + "strings" "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" ) const apiVersion = "7.1" @@ -45,7 +47,7 @@ type Client struct { func NewClient(con *models.AzuredevopsConnection, apiClient plugin.ApiClient, url string) Client { return Client{ c: http.Client{ - Timeout: 2 * time.Second, + Timeout: 10 * time.Second, }, connection: con, url: url, @@ -54,18 +56,27 @@ func NewClient(con *models.AzuredevopsConnection, apiClient plugin.ApiClient, ur } func (c *Client) GetUserProfile() (Profile, errors.Error) { + // On-Premises Azure DevOps Server does not have the global VSSPS profile API. + // Return a placeholder profile to avoid blocking the connection test flow. + if c.connection != nil && c.connection.Endpoint != "" { + return Profile{ + DisplayName: "On-Premises User", + }, nil + } + var p Profile - endpoint, err := url.JoinPath(c.url, "/_apis/profile/profiles/me") - if err != nil { - return Profile{}, errors.Internal.Wrap(err, "failed to join user profile path") + baseUrl := strings.TrimRight(c.url, "/") + if baseUrl == "" { + baseUrl = "https://app.vssps.visualstudio.com" } + endpoint := fmt.Sprintf("%s/_apis/profile/profiles/me?api-version=7.1-preview.1", baseUrl) res, err := c.doGet(endpoint) if err != nil { return Profile{}, errors.Internal.Wrap(err, "failed to read user accounts") } - if res.StatusCode == 203 || res.StatusCode == 401 { + if res.StatusCode == 203 || res.StatusCode == 302 || res.StatusCode == 401 { return Profile{}, errors.Unauthorized.New("failed to read user profile") } @@ -76,14 +87,19 @@ func (c *Client) GetUserProfile() (Profile, errors.Error) { } if err := json.Unmarshal(resBody, &p); err != nil { - panic(err) + return Profile{}, errors.Internal.Wrap(err, "failed to unmarshal user profile") } return p, nil } func (c *Client) GetUserAccounts(memberId string) (AccountResponse, errors.Error) { + // On-Premises installations do not have the global accounts API, return empty list. + if c.connection != nil && c.connection.Endpoint != "" { + return AccountResponse{}, nil + } + var a AccountResponse - endpoint := fmt.Sprintf(c.url+"/_apis/accounts?memberId=%s", memberId) + endpoint := fmt.Sprintf("%s/_apis/accounts?memberId=%s&api-version=7.1-preview.1", strings.TrimRight(c.url, "/"), memberId) res, err := c.doGet(endpoint) if err != nil { return nil, errors.Internal.Wrap(err, "failed to read user accounts") @@ -114,7 +130,7 @@ func (c *Client) doGet(url string) (*http.Response, error) { if err = c.connection.GetAccessTokenAuthenticator().SetupAuthentication(req); err != nil { return nil, errors.Internal.Wrap(err, "failed to authorize the request using the plugin connection") } - return http.DefaultClient.Do(req) + return c.c.Do(req) } type GetProjectsArgs struct { @@ -147,7 +163,10 @@ func (c *Client) GetProjects(args GetProjectsArgs) ([]Project, errors.Error) { query.Set("$top", strconv.Itoa(top)) query.Set("$skip", strconv.Itoa(skip)) - path := fmt.Sprintf("%s/_apis/projects", args.OrgId) + path := "_apis/projects" + if args.OrgId != "" { + path = fmt.Sprintf("%s/_apis/projects", args.OrgId) + } res, err := c.apiClient.Get(path, query, nil) if err != nil { return nil, err @@ -190,6 +209,9 @@ func (c *Client) GetRepositories(args GetRepositoriesArgs) ([]Repository, errors } path := fmt.Sprintf("%s/%s/_apis/git/repositories", args.OrgId, args.ProjectId) + if args.OrgId == "" { + path = fmt.Sprintf("%s/_apis/git/repositories", args.ProjectId) + } res, err := c.apiClient.Get(path, query, nil) if err != nil { return nil, err diff --git a/backend/plugins/azuredevops_go/api/azuredevops/client_onprem_test.go b/backend/plugins/azuredevops_go/api/azuredevops/client_onprem_test.go new file mode 100644 index 00000000000..b2b766809b3 --- /dev/null +++ b/backend/plugins/azuredevops_go/api/azuredevops/client_onprem_test.go @@ -0,0 +1,107 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azuredevops + +import ( + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" + "testing" +) + +func TestGetUserProfile_OnPremises(t *testing.T) { + conn := &models.AzuredevopsConnection{ + BaseConnection: api.BaseConnection{}, + AzuredevopsConn: models.AzuredevopsConn{ + AzuredevopsAccessToken: models.AzuredevopsAccessToken{ + Token: "test-token", + }, + Endpoint: "https://tfs.company.local/DefaultCollection/", + }, + } + + client := NewClient(conn, nil, "https://tfs.company.local/DefaultCollection/") + profile, err := client.GetUserProfile() + if err != nil { + t.Fatalf("GetUserProfile() returned unexpected error for On-Premises: %v", err) + } + if profile.DisplayName != "On-Premises User" { + t.Errorf("GetUserProfile() DisplayName = %q; want %q", profile.DisplayName, "On-Premises User") + } +} + +func TestGetUserAccounts_OnPremises(t *testing.T) { + conn := &models.AzuredevopsConnection{ + BaseConnection: api.BaseConnection{}, + AzuredevopsConn: models.AzuredevopsConn{ + AzuredevopsAccessToken: models.AzuredevopsAccessToken{ + Token: "test-token", + }, + Endpoint: "https://tfs.company.local/DefaultCollection/", + }, + } + + client := NewClient(conn, nil, "https://tfs.company.local/DefaultCollection/") + accounts, err := client.GetUserAccounts("test-member-id") + if err != nil { + t.Fatalf("GetUserAccounts() returned unexpected error for On-Premises: %v", err) + } + if len(accounts) != 0 { + t.Errorf("GetUserAccounts() returned %d accounts; want 0 for On-Premises", len(accounts)) + } +} + +func TestGetUserProfile_Cloud(t *testing.T) { + // For Cloud connections (no Endpoint set), GetUserProfile should NOT return + // the placeholder profile. It should attempt to call the VSSPS API. + conn := &models.AzuredevopsConnection{ + BaseConnection: api.BaseConnection{}, + AzuredevopsConn: models.AzuredevopsConn{ + AzuredevopsAccessToken: models.AzuredevopsAccessToken{ + Token: "test-token", + }, + // Endpoint is empty = Cloud mode + }, + } + + // Without a mock server, this will fail with a connection error, + // which is expected - the important thing is it does NOT bypass to placeholder + client := NewClient(conn, nil, "http://localhost:0") + _, err := client.GetUserProfile() + if err == nil { + t.Error("GetUserProfile() for Cloud should attempt real API call and fail without a server") + } +} + +func TestGetUserAccounts_Cloud(t *testing.T) { + conn := &models.AzuredevopsConnection{ + BaseConnection: api.BaseConnection{}, + AzuredevopsConn: models.AzuredevopsConn{ + AzuredevopsAccessToken: models.AzuredevopsAccessToken{ + Token: "test-token", + }, + // Endpoint is empty = Cloud mode + }, + } + + // Without a mock server, this will fail with a connection error + client := NewClient(conn, nil, "http://localhost:0") + _, err := client.GetUserAccounts("test-member-id") + if err == nil { + t.Error("GetUserAccounts() for Cloud should attempt real API call and fail without a server") + } +} diff --git a/backend/plugins/azuredevops_go/api/blueprint_v200.go b/backend/plugins/azuredevops_go/api/blueprint_v200.go index 018d9d1d77e..3cd4acb711b 100644 --- a/backend/plugins/azuredevops_go/api/blueprint_v200.go +++ b/backend/plugins/azuredevops_go/api/blueprint_v200.go @@ -180,7 +180,11 @@ func makePipelinePlanV200( } if scope.Scope.Type == models.RepositoryTypeADO { - cloneUrl.User = url.UserPassword("git", connection.Token) + username := connection.Username + if username == "" { + username = "git" + } + cloneUrl.User = url.UserPassword(username, connection.Token) } stage = append(stage, &coreModels.PipelineTask{ Plugin: "gitextractor", diff --git a/backend/plugins/azuredevops_go/api/connection_api.go b/backend/plugins/azuredevops_go/api/connection_api.go index 7da66cd1fe5..81123503436 100644 --- a/backend/plugins/azuredevops_go/api/connection_api.go +++ b/backend/plugins/azuredevops_go/api/connection_api.go @@ -19,13 +19,15 @@ package api import ( "context" + "fmt" + "net/http" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/azuredevops_go/api/azuredevops" "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" "github.com/apache/incubator-devlake/server/api/shared" - "net/http" ) type AzuredevopsTestConnResponse struct { @@ -55,7 +57,7 @@ func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, } body, err := testConnection(context.TODO(), connection) if err != nil { - return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + return nil, errors.BadInput.Wrap(err, err.Error()) } return &plugin.ApiResourceOutput{Body: body, Status: http.StatusOK}, nil } @@ -76,7 +78,7 @@ func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResource body, err := testConnection(context.TODO(), *connection) if err != nil { - return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + return nil, errors.BadInput.Wrap(err, err.Error()) } return &plugin.ApiResourceOutput{Body: body, Status: http.StatusOK}, nil } @@ -155,20 +157,39 @@ func testConnection(ctx context.Context, connection models.AzuredevopsConnection if err != nil { return nil, err } - - vsc := azuredevops.NewClient(&connection, apiClient, "https://app.vssps.visualstudio.com/") - org := connection.Organization - - if org == "" { - _, err = vsc.GetUserProfile() - } else { + if connection.Endpoint != "" { + // On-Premises: Test connection by directly fetching projects from On-Premises server endpoint + vsc := azuredevops.NewClient(&connection, apiClient, connection.Endpoint) args := azuredevops.GetProjectsArgs{ - OrgId: org, + OrgId: connection.Organization, } _, err = vsc.GetProjects(args) - } - if err != nil { - return nil, err + if err != nil { + // On-Premises Azure DevOps Server HTTP endpoints may use Windows NTLM Auth (which Git CLI succeeds with). + // If GetProjects returns 401/Unauthorized, log a warning but allow connection test to succeed. + basicRes.GetLogger().Warn(err, "GetProjects returned error for On-Premises connection, proceeding with connection setup") + } + } else { + // Cloud: Azure DevOps Cloud test logic + org := connection.Organization + if org != "" { + // Organization is specified: use dev.azure.com directly (works with org-scoped PATs too) + vsc := azuredevops.NewClient(&connection, apiClient, "https://dev.azure.com/") + args := azuredevops.GetProjectsArgs{ + OrgId: org, + } + _, err = vsc.GetProjects(args) + if err != nil { + return nil, errors.BadInput.Wrap(err, fmt.Sprintf("Failed to fetch projects for Azure DevOps Cloud Organization '%s'. Please check your Organization name and PAT token permissions.", org)) + } + } else { + // No organization specified: try VSSPS global profile (requires 'All accessible organizations' PAT) + vsc := azuredevops.NewClient(&connection, apiClient, "https://app.vssps.visualstudio.com/") + _, profileErr := vsc.GetUserProfile() + if profileErr != nil { + return nil, errors.BadInput.New("Azure DevOps Cloud authentication failed. If your PAT token was created for a specific Organization, please enter your Organization name in the form. Otherwise ensure your PAT was created with 'All accessible organizations'.") + } + } } connection = connection.Sanitize() diff --git a/backend/plugins/azuredevops_go/api/remote_helper.go b/backend/plugins/azuredevops_go/api/remote_helper.go index 00110daf673..ba3fa875d6d 100644 --- a/backend/plugins/azuredevops_go/api/remote_helper.go +++ b/backend/plugins/azuredevops_go/api/remote_helper.go @@ -59,31 +59,48 @@ func listAzuredevopsRemoteScopes( ) { org := connection.Organization - vsc := azuredevops.NewClient(connection, apiClient, "https://app.vssps.visualstudio.com") + // Use the connection endpoint for On-Premises, or VSSPS for Cloud + clientUrl := "https://app.vssps.visualstudio.com" + if connection.Endpoint != "" { + clientUrl = connection.GetEndpoint() + } + vsc := azuredevops.NewClient(connection, apiClient, clientUrl) if groupId == "" { - return listAzuredevopsProjects(vsc, page, org) + return listAzuredevopsProjects(connection, vsc, page, org) } id := strings.Split(groupId, idSeparator) - - if remote, err := listRemoteRepos(vsc, id[0], id[1]); err == nil { - children = append(children, remote...) + orgId := id[0] + projectId := "" + if len(id) > 1 { + projectId = id[1] + } else if connection.Endpoint != "" { + orgId = "" + projectId = id[0] } - if remote, err := listAzuredevopsRepos(vsc, id[0], id[1]); err == nil { - children = append(children, remote...) + if projectId != "" { + if remote, err := listRemoteRepos(vsc, orgId, projectId); err == nil { + children = append(children, remote...) + } + + if remote, err := listAzuredevopsRepos(vsc, orgId, projectId); err == nil { + children = append(children, remote...) + } } return children, nextPage, nil } -func listAzuredevopsProjects(vsc azuredevops.Client, _ AzuredevopsRemotePagination, org string) ( +func listAzuredevopsProjects(connection *models.AzuredevopsConnection, vsc azuredevops.Client, _ AzuredevopsRemotePagination, org string) ( children []dsmodels.DsRemoteApiScopeListEntry[models.AzuredevopsRepo], nextPage *AzuredevopsRemotePagination, err errors.Error) { var accounts azuredevops.AccountResponse - if org == "" { + if connection.Endpoint != "" { + accounts = append(accounts, azuredevops.Account{AccountName: org}) + } else if org == "" { profile, err := vsc.GetUserProfile() if err != nil { return nil, nil, err @@ -114,8 +131,12 @@ func listAzuredevopsProjects(vsc azuredevops.Client, _ AzuredevopsRemotePaginati var tmp []dsmodels.DsRemoteApiScopeListEntry[models.AzuredevopsRepo] for _, vv := range projects { + groupId := vv.Name + if accountName != "" { + groupId = accountName + idSeparator + vv.Name + } tmp = append(tmp, dsmodels.DsRemoteApiScopeListEntry[models.AzuredevopsRepo]{ - Id: accountName + idSeparator + vv.Name, + Id: groupId, Type: api.RAS_ENTRY_TYPE_GROUP, Name: vv.Name, }) @@ -153,7 +174,10 @@ func listAzuredevopsRepos( continue } - pID := orgId + idSeparator + projectId + pID := projectId + if orgId != "" { + pID = orgId + idSeparator + projectId + } repo := models.AzuredevopsRepo{ Id: v.Id, Type: models.RepositoryTypeADO, diff --git a/backend/plugins/azuredevops_go/models/connection.go b/backend/plugins/azuredevops_go/models/connection.go index 3bb9f4a9614..a2af72be222 100644 --- a/backend/plugins/azuredevops_go/models/connection.go +++ b/backend/plugins/azuredevops_go/models/connection.go @@ -25,6 +25,7 @@ import ( "github.com/apache/incubator-devlake/core/utils" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "net/http" + "strings" ) var _ plugin.ApiConnection = (*AzuredevopsConn)(nil) @@ -50,16 +51,39 @@ func (at *AzuredevopsAccessToken) SetupAuthentication(req *http.Request) errors. // AzuredevopsConn holds the essential information to connect to the Azure DevOps API type AzuredevopsConn struct { - //api.RestConnection `mapstructure:",squash"` AzuredevopsAccessToken `mapstructure:",squash"` Organization string `json:"organization"` - //Endpoint string `mapstructure:"endpoint" json:"endpoint"` - Proxy string `mapstructure:"proxy" json:"proxy"` - //RateLimitPerHour int `comment:"api request rate limit per hour" json:"rateLimitPerHour"` + Username string `mapstructure:"username" json:"username"` + Proxy string `mapstructure:"proxy" json:"proxy"` + + // Endpoint is the base URL for On-Premises Azure DevOps Server (optional, empty for Cloud) + Endpoint string `mapstructure:"endpoint" json:"endpoint" validate:"omitempty,url"` +} + +func (conn *AzuredevopsConn) GetAccessTokenAuthenticator() plugin.ApiAuthenticator { + return conn +} + +func (conn *AzuredevopsConn) SetupAuthentication(req *http.Request) errors.Error { + username := conn.Username + if conn.Endpoint != "" { + // On-Premises Azure DevOps REST API uses PAT token auth with empty username over HTTP Basic Auth + username = "" + } + req.SetBasicAuth(username, conn.Token) + return nil } func (conn *AzuredevopsConn) GetEndpoint() string { - return "https://dev.azure.com" + // Returns the On-Premises endpoint if configured, otherwise defaults to Azure DevOps Cloud URL + if conn.Endpoint != "" { + ep := conn.Endpoint + if !strings.HasSuffix(ep, "/") { + ep += "/" + } + return ep + } + return "https://dev.azure.com/" } func (conn *AzuredevopsConn) GetProxy() string { @@ -79,9 +103,9 @@ type AzuredevopsConnection struct { } func (c AzuredevopsConnection) GetEndpoint() string { - return "https://dev.azure.com" + // Delegates to the embedded AzuredevopsConn.GetEndpoint() + return c.AzuredevopsConn.GetEndpoint() } - func (c AzuredevopsConnection) GetProxy() string { return c.Proxy } diff --git a/backend/plugins/azuredevops_go/models/connection_test.go b/backend/plugins/azuredevops_go/models/connection_test.go new file mode 100644 index 00000000000..10f265d11ac --- /dev/null +++ b/backend/plugins/azuredevops_go/models/connection_test.go @@ -0,0 +1,144 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "net/http" + "testing" +) + +func TestGetEndpoint_CloudDefault(t *testing.T) { + conn := &AzuredevopsConn{} + endpoint := conn.GetEndpoint() + if endpoint != "https://dev.azure.com/" { + t.Errorf("GetEndpoint() = %q; want %q", endpoint, "https://dev.azure.com/") + } +} + +func TestGetEndpoint_OnPremises(t *testing.T) { + conn := &AzuredevopsConn{ + Endpoint: "https://tfs.company.local/DefaultCollection/", + } + endpoint := conn.GetEndpoint() + if endpoint != "https://tfs.company.local/DefaultCollection/" { + t.Errorf("GetEndpoint() = %q; want %q", endpoint, "https://tfs.company.local/DefaultCollection/") + } +} + +func TestGetEndpoint_TrailingSlash(t *testing.T) { + conn := &AzuredevopsConn{ + Endpoint: "https://tfs.company.local/DefaultCollection", + } + endpoint := conn.GetEndpoint() + expected := "https://tfs.company.local/DefaultCollection/" + if endpoint != expected { + t.Errorf("GetEndpoint() = %q; want %q (trailing slash should be appended)", endpoint, expected) + } +} + +func TestGetEndpoint_ConnectionWrapper(t *testing.T) { + conn := AzuredevopsConnection{ + AzuredevopsConn: AzuredevopsConn{ + Endpoint: "https://tfs.company.local/DefaultCollection/", + }, + } + endpoint := conn.GetEndpoint() + if endpoint != "https://tfs.company.local/DefaultCollection/" { + t.Errorf("AzuredevopsConnection.GetEndpoint() = %q; want %q", endpoint, "https://tfs.company.local/DefaultCollection/") + } +} + +func TestGetEndpoint_ConnectionWrapperCloud(t *testing.T) { + conn := AzuredevopsConnection{} + endpoint := conn.GetEndpoint() + if endpoint != "https://dev.azure.com/" { + t.Errorf("AzuredevopsConnection.GetEndpoint() = %q; want %q", endpoint, "https://dev.azure.com/") + } +} + +func TestSetupAuthentication_Cloud(t *testing.T) { + conn := &AzuredevopsConn{ + AzuredevopsAccessToken: AzuredevopsAccessToken{ + Token: "test-pat-token", + }, + Username: "testuser", + } + + req, _ := http.NewRequest("GET", "https://dev.azure.com/org/_apis/projects", nil) + err := conn.SetupAuthentication(req) + if err != nil { + t.Fatalf("SetupAuthentication() returned error: %v", err) + } + + user, pass, ok := req.BasicAuth() + if !ok { + t.Fatal("SetupAuthentication() did not set Basic Auth header") + } + if user != "testuser" { + t.Errorf("Basic Auth username = %q; want %q", user, "testuser") + } + if pass != "test-pat-token" { + t.Errorf("Basic Auth password = %q; want %q", pass, "test-pat-token") + } +} + +func TestSetupAuthentication_OnPremises(t *testing.T) { + conn := &AzuredevopsConn{ + AzuredevopsAccessToken: AzuredevopsAccessToken{ + Token: "on-prem-pat-token", + }, + Username: "domain\\admin", + Endpoint: "https://tfs.company.local/DefaultCollection/", + } + + req, _ := http.NewRequest("GET", "https://tfs.company.local/DefaultCollection/_apis/projects", nil) + err := conn.SetupAuthentication(req) + if err != nil { + t.Fatalf("SetupAuthentication() returned error: %v", err) + } + + user, pass, ok := req.BasicAuth() + if !ok { + t.Fatal("SetupAuthentication() did not set Basic Auth header") + } + // On-Premises: username should be empty (PAT auth with empty username) + if user != "" { + t.Errorf("On-Premises Basic Auth username = %q; want empty string", user) + } + if pass != "on-prem-pat-token" { + t.Errorf("On-Premises Basic Auth password = %q; want %q", pass, "on-prem-pat-token") + } +} + +func TestSanitize(t *testing.T) { + conn := &AzuredevopsConn{ + AzuredevopsAccessToken: AzuredevopsAccessToken{ + Token: "secret-token", + }, + Username: "testuser", + Endpoint: "https://tfs.company.local/", + } + + sanitized := conn.Sanitize() + if sanitized.Endpoint != "https://tfs.company.local/" { + t.Errorf("Sanitize() should preserve Endpoint, got %q", sanitized.Endpoint) + } + if sanitized.Username != "testuser" { + t.Errorf("Sanitize() should preserve Username, got %q", sanitized.Username) + } +} diff --git a/backend/plugins/azuredevops_go/models/migrationscripts/20260722_add_endpoint_to_azuredevops.go b/backend/plugins/azuredevops_go/models/migrationscripts/20260722_add_endpoint_to_azuredevops.go new file mode 100644 index 00000000000..ae71e067605 --- /dev/null +++ b/backend/plugins/azuredevops_go/models/migrationscripts/20260722_add_endpoint_to_azuredevops.go @@ -0,0 +1,49 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type addEndpointToAzuredevops struct{} + +type azuredevopsConnection20260722 struct { + Endpoint string `gorm:"type:varchar(255)"` +} + +func (azuredevopsConnection20260722) TableName() string { + return "_tool_azuredevops_go_connections" +} + +func (script *addEndpointToAzuredevops) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &azuredevopsConnection20260722{}, + ) +} + +func (*addEndpointToAzuredevops) Version() uint64 { + return 20260722000002 +} + +func (*addEndpointToAzuredevops) Name() string { + return "add endpoint field to _tool_azuredevops_go_connections" +} diff --git a/backend/plugins/azuredevops_go/models/migrationscripts/20260726_add_username_to_azuredevops.go b/backend/plugins/azuredevops_go/models/migrationscripts/20260726_add_username_to_azuredevops.go new file mode 100644 index 00000000000..c35ad48c806 --- /dev/null +++ b/backend/plugins/azuredevops_go/models/migrationscripts/20260726_add_username_to_azuredevops.go @@ -0,0 +1,49 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type addUsernameToAzuredevops struct{} + +type azuredevopsConnection20260726 struct { + Username string `gorm:"type:varchar(255)"` +} + +func (azuredevopsConnection20260726) TableName() string { + return "_tool_azuredevops_go_connections" +} + +func (script *addUsernameToAzuredevops) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &azuredevopsConnection20260726{}, + ) +} + +func (*addUsernameToAzuredevops) Version() uint64 { + return 20260726000001 +} + +func (*addUsernameToAzuredevops) Name() string { + return "add username field to _tool_azuredevops_go_connections" +} diff --git a/backend/plugins/azuredevops_go/models/migrationscripts/archived/connection.go b/backend/plugins/azuredevops_go/models/migrationscripts/archived/connection.go index 18e9a6266e3..9bbeb0a455d 100644 --- a/backend/plugins/azuredevops_go/models/migrationscripts/archived/connection.go +++ b/backend/plugins/azuredevops_go/models/migrationscripts/archived/connection.go @@ -28,6 +28,7 @@ type AzuredevopsConnection struct { Token string `mapstructure:"token" validate:"required" encrypt:"yes"` Proxy string `gorm:"type:varchar(255)"` Organization string `gorm:"type:varchar(255)"` + Endpoint string `gorm:"type:varchar(255)"` } func (AzuredevopsConnection) TableName() string { diff --git a/backend/plugins/azuredevops_go/models/migrationscripts/register.go b/backend/plugins/azuredevops_go/models/migrationscripts/register.go index 59fa7832fe0..cd08322ae83 100644 --- a/backend/plugins/azuredevops_go/models/migrationscripts/register.go +++ b/backend/plugins/azuredevops_go/models/migrationscripts/register.go @@ -26,5 +26,7 @@ func All() []plugin.MigrationScript { return []plugin.MigrationScript{ new(addInitTables), new(extendRepoTable), + new(addEndpointToAzuredevops), + new(addUsernameToAzuredevops), } } diff --git a/backend/plugins/azuredevops_go/tasks/account_collector.go b/backend/plugins/azuredevops_go/tasks/account_collector.go index 08942679a66..f326bde7ac8 100644 --- a/backend/plugins/azuredevops_go/tasks/account_collector.go +++ b/backend/plugins/azuredevops_go/tasks/account_collector.go @@ -43,6 +43,11 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error { rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, rawUserTable) logger := taskCtx.GetLogger() + if data.Options.OrganizationId == "" { + logger.Info("OrganizationId is empty or On-Premises mode, skipping Cloud Graph user collection") + return nil + } + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ RawDataSubTaskArgs: *rawDataSubTaskArgs, ApiClient: data.ApiClient, diff --git a/backend/plugins/azuredevops_go/tasks/ci_cd_build_collector.go b/backend/plugins/azuredevops_go/tasks/ci_cd_build_collector.go index 257c5fac0fb..72e9f141b4a 100644 --- a/backend/plugins/azuredevops_go/tasks/ci_cd_build_collector.go +++ b/backend/plugins/azuredevops_go/tasks/ci_cd_build_collector.go @@ -63,7 +63,7 @@ func CollectBuilds(taskCtx plugin.SubTaskContext) errors.Error { GetNextPageCustomData: ExtractContToken, PageSize: 100, FinalizableApiCollectorCommonArgs: api.FinalizableApiCollectorCommonArgs{ - UrlTemplate: "{{ .Params.OrganizationId }}/{{ .Params.ProjectId }}/_apis/build/builds?api-version=7.1", + UrlTemplate: "{{ if .Params.OrganizationId }}{{ .Params.OrganizationId }}/{{ end }}{{ .Params.ProjectId }}/_apis/build/builds?api-version=7.1", Query: func(reqData *api.RequestData, createdAfter *time.Time) (url.Values, errors.Error) { query := url.Values{} query.Set("repositoryType", repoType) diff --git a/backend/plugins/azuredevops_go/tasks/ci_cd_timeline_records_collector.go b/backend/plugins/azuredevops_go/tasks/ci_cd_timeline_records_collector.go index 9ac7669178e..36643d6451d 100644 --- a/backend/plugins/azuredevops_go/tasks/ci_cd_timeline_records_collector.go +++ b/backend/plugins/azuredevops_go/tasks/ci_cd_timeline_records_collector.go @@ -65,7 +65,7 @@ func CollectRecords(taskCtx plugin.SubTaskContext) errors.Error { ApiClient: data.ApiClient, Input: iterator, Incremental: false, - UrlTemplate: "{{ .Params.OrganizationId }}/{{ .Params.ProjectId }}/_apis/build/builds/{{ .Input.AzuredevopsId }}/Timeline?api-version=7.1", + UrlTemplate: "{{ if .Params.OrganizationId }}{{ .Params.OrganizationId }}/{{ end }}{{ .Params.ProjectId }}/_apis/build/builds/{{ .Input.AzuredevopsId }}/Timeline?api-version=7.1", Query: BuildPaginator(true), ResponseParser: ParseRawMessageFromRecords, AfterResponse: ignoreInvalidTimelineResponse, // Skip builds with missing/malformed timelines (e.g. YAML syntax errors) diff --git a/backend/plugins/azuredevops_go/tasks/commit_collector.go b/backend/plugins/azuredevops_go/tasks/commit_collector.go index fe27de1eca9..64a0af9d546 100644 --- a/backend/plugins/azuredevops_go/tasks/commit_collector.go +++ b/backend/plugins/azuredevops_go/tasks/commit_collector.go @@ -46,7 +46,7 @@ func CollectApiCommits(taskCtx plugin.SubTaskContext) errors.Error { ApiClient: data.ApiClient, PageSize: 100, Incremental: false, - UrlTemplate: "{{ .Params.OrganizationId }}/{{ .Params.ProjectId }}/_apis/git/repositories/{{ .Params.RepositoryId }}/commits?api-version=7.1", + UrlTemplate: "{{ if .Params.OrganizationId }}{{ .Params.OrganizationId }}/{{ end }}{{ .Params.ProjectId }}/_apis/git/repositories/{{ .Params.RepositoryId }}/commits?api-version=7.1", Query: BuildPaginator(false), ResponseParser: ParseRawMessageFromValue, AfterResponse: change203To401, diff --git a/backend/plugins/azuredevops_go/tasks/pr_collector.go b/backend/plugins/azuredevops_go/tasks/pr_collector.go index 30a63dba7bb..e9a306f81d0 100644 --- a/backend/plugins/azuredevops_go/tasks/pr_collector.go +++ b/backend/plugins/azuredevops_go/tasks/pr_collector.go @@ -61,7 +61,7 @@ func CollectApiPullRequests(taskCtx plugin.SubTaskContext) errors.Error { RawDataSubTaskArgs: *rawDataSubTaskArgs, ApiClient: data.ApiClient, PageSize: 100, - UrlTemplate: "{{ .Params.OrganizationId }}/{{ .Params.ProjectId }}/_apis/git/repositories/{{ .Params.RepositoryId }}/pullrequests?api-version=7.1", + UrlTemplate: "{{ if .Params.OrganizationId }}{{ .Params.OrganizationId }}/{{ end }}{{ .Params.ProjectId }}/_apis/git/repositories/{{ .Params.RepositoryId }}/pullrequests?api-version=7.1", Query: func(reqData *api.RequestData) (url.Values, errors.Error) { query := url.Values{} query.Set("searchCriteria.status", "all") diff --git a/backend/plugins/azuredevops_go/tasks/pr_commit_collector.go b/backend/plugins/azuredevops_go/tasks/pr_commit_collector.go index dbbddd67b7d..a7a7c00dcec 100644 --- a/backend/plugins/azuredevops_go/tasks/pr_commit_collector.go +++ b/backend/plugins/azuredevops_go/tasks/pr_commit_collector.go @@ -71,7 +71,7 @@ func CollectApiPullRequestCommits(taskCtx plugin.SubTaskContext) errors.Error { PageSize: 100, Input: iterator, Incremental: false, - UrlTemplate: "{{ .Params.OrganizationId }}/{{ .Params.ProjectId }}/_apis/git/repositories/{{ .Params.RepositoryId }}/pullRequests/{{ .Input.AzuredevopsId }}/commits?api-version=7.1", + UrlTemplate: "{{ if .Params.OrganizationId }}{{ .Params.OrganizationId }}/{{ end }}{{ .Params.ProjectId }}/_apis/git/repositories/{{ .Params.RepositoryId }}/pullRequests/{{ .Input.AzuredevopsId }}/commits?api-version=7.1", Query: BuildPaginator(true), ResponseParser: ParseRawMessageFromValue, GetNextPageCustomData: ExtractContToken, diff --git a/backend/plugins/gitextractor/parser/clone_gitcli.go b/backend/plugins/gitextractor/parser/clone_gitcli.go index 61931b521b5..9d0ac2ed27a 100644 --- a/backend/plugins/gitextractor/parser/clone_gitcli.go +++ b/backend/plugins/gitextractor/parser/clone_gitcli.go @@ -18,6 +18,7 @@ limitations under the License. package parser import ( + "encoding/base64" "fmt" "net/url" "os" @@ -46,16 +47,17 @@ type CloneRepoConfig struct { } type GitcliCloner struct { - ctx plugin.SubTaskContext - taskData *GitExtractorTaskData - logger log.Logger - stateManager *api.SubtaskStateManager - since *time.Time - remoteUrl string - localDir string - success bool - syncEnvs []string - syncArgs []string + ctx plugin.SubTaskContext + taskData *GitExtractorTaskData + logger log.Logger + stateManager *api.SubtaskStateManager + since *time.Time + remoteUrl string + localDir string + success bool + syncEnvs []string + syncArgs []string + globalConfigArgs []string } func NewGitcliCloner(ctx plugin.SubTaskContext, localDir string) (*GitcliCloner, errors.Error) { @@ -93,8 +95,17 @@ func (g *GitcliCloner) prepareSync() errors.Error { if e != nil { return errors.Convert(e) } - // support proxy + // support proxy and http auth if remoteUrl.Scheme == "http" || remoteUrl.Scheme == "https" { + g.globalConfigArgs = append(g.globalConfigArgs, "-c", "credential.helper=") + if remoteUrl.User != nil { + password, ok := remoteUrl.User.Password() + if ok && password != "" { + username := remoteUrl.User.Username() + auth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", username, password))) + g.globalConfigArgs = append(g.globalConfigArgs, "-c", fmt.Sprintf("http.extraHeader=Authorization: Basic %s", auth)) + } + } if taskData.Options.Proxy != "" { g.syncEnvs = append(g.syncEnvs, fmt.Sprintf("HTTPS_PROXY=%s", taskData.Options.Proxy)) } @@ -308,8 +319,10 @@ func (g *GitcliCloner) gitCmd(gitcmd string, args ...string) errors.Error { func (g *GitcliCloner) git(env []string, dir string, gitcmd string, args ...string) errors.Error { g.logger.Debug("git %s %v", gitcmd, sanitizeArgs(args)) // CWE-532: sanitize before logging - args = append([]string{gitcmd}, args...) - cmd := exec.CommandContext(g.ctx.GetContext(), "git", args...) + cmdArgs := append([]string{}, g.globalConfigArgs...) + cmdArgs = append(cmdArgs, gitcmd) + cmdArgs = append(cmdArgs, args...) + cmd := exec.CommandContext(g.ctx.GetContext(), "git", cmdArgs...) cmd.Env = env cmd.Dir = dir return g.execCommand(cmd) diff --git a/config-ui/src/plugins/components/data-scope-remote/search-local.tsx b/config-ui/src/plugins/components/data-scope-remote/search-local.tsx index a84fbc494f4..a1e382bf013 100644 --- a/config-ui/src/plugins/components/data-scope-remote/search-local.tsx +++ b/config-ui/src/plugins/components/data-scope-remote/search-local.tsx @@ -102,7 +102,7 @@ export const SearchLocal = ({ mode, plugin, connectionId, config, disabledScope, pageToken: currentPageToken, }); - newItems = (res.children ?? []).map((it) => ({ + newItems = (res?.children ?? []).map((it) => ({ ...it, title: it.name, })); diff --git a/config-ui/src/plugins/components/data-scope-remote/search-remote.tsx b/config-ui/src/plugins/components/data-scope-remote/search-remote.tsx index 5f61d74106b..2a23a66d3cf 100644 --- a/config-ui/src/plugins/components/data-scope-remote/search-remote.tsx +++ b/config-ui/src/plugins/components/data-scope-remote/search-remote.tsx @@ -96,7 +96,7 @@ export const SearchRemote = ({ mode, plugin, connectionId, config, disabledScope pageToken: currentPageToken, }); - newItems = (res.children ?? []).map((it) => ({ + newItems = (res?.children ?? []).map((it) => ({ ...it, title: getPluginScopeName(plugin, it) || it.name, })); @@ -140,7 +140,7 @@ export const SearchRemote = ({ mode, plugin, connectionId, config, disabledScope pageSize: PAGE_SIZE, }); - const newItems = (res.children ?? []).map((it) => ({ + const newItems = (res?.children ?? []).map((it) => ({ ...it, title: getPluginScopeName(plugin, it) || it.fullName || it.name, })); diff --git a/config-ui/src/plugins/components/data-scope-select/index.tsx b/config-ui/src/plugins/components/data-scope-select/index.tsx index 860cce6cd06..2401c37b6ec 100644 --- a/config-ui/src/plugins/components/data-scope-select/index.tsx +++ b/config-ui/src/plugins/components/data-scope-select/index.tsx @@ -67,7 +67,7 @@ export const DataScopeSelect = ({ const res = await API.scope.list(plugin, connectionId, { page, pageSize }); setItems((items) => [ ...items, - ...res.scopes.map((sc) => ({ + ...(res.scopes ?? []).map((sc) => ({ parentId: null, id: getPluginScopeId(plugin, sc.scope), title: getPluginScopeName(plugin, sc.scope) || sc.scope.fullName || sc.scope.name, @@ -92,10 +92,10 @@ export const DataScopeSelect = ({ const searchOptions = useMemo( () => - data?.scopes.map((sc) => ({ + (data?.scopes ?? []).map((sc) => ({ label: getPluginScopeName(plugin, sc.scope) || sc.scope.fullName || sc.scope.name, value: getPluginScopeId(plugin, sc.scope), - })) ?? [], + })), [data], ); diff --git a/config-ui/src/plugins/register/azure/config.tsx b/config-ui/src/plugins/register/azure/config.tsx index a3f574d615a..978a6af0934 100644 --- a/config-ui/src/plugins/register/azure/config.tsx +++ b/config-ui/src/plugins/register/azure/config.tsx @@ -33,7 +33,9 @@ export const AzureConfig: IPluginConfig = { initialValues: {}, fields: [ 'name', - () => , + ({ values, setValues }: any) => ( + setValues({ endpoint: val })} /> + ), { key: 'token', label: 'Personal Access Token', @@ -86,10 +88,24 @@ export const AzureGoConfig: IPluginConfig = { initialValues: {}, fields: [ 'name', - () => , + ({ values, setValues }: any) => ( + setValues({ endpoint: val })} /> + ), + { + key: 'username', + label: 'Username (Optional for On-Premises)', + subLabel: + 'For On-Premises Azure DevOps Server, enter your Windows/Domain username if required (e.g. Administrator or domain\\user). Leave empty for Cloud.', + }, { key: 'token', label: 'Personal Access Token', + subLabel: ( + + For Azure DevOps Cloud, use a Personal Access Token (PAT). For On-Premises Server, use a PAT or your domain + password. + + ), }, ({ initialValues, values, setValues }: any) => ( { +interface Props { + value?: string; + onChange?: (value: string) => void; +} + +export const BaseURL = ({ value, onChange }: Props) => { + // Default to 'server' mode if an endpoint value already exists, 'cloud' otherwise + const [version, setVersion] = useState<'cloud' | 'server'>(value ? 'server' : 'cloud'); + + const handleVersionChange = (e: any) => { + const selectedVersion = e.target.value; + setVersion(selectedVersion); + if (selectedVersion === 'cloud') { + onChange?.(''); // Reset endpoint when switching to Cloud mode + } + }; + return ( - {}}> + Azure DevOps Cloud - - Azure DevOps Server (not supported) - + Azure DevOps Server (On-Premises) -

If you are using Azure DevOps Cloud, you do not need to enter the endpoint URL.

+ + {version === 'cloud' ? ( +

+ If you are using Azure DevOps Cloud, you do not need to enter the endpoint URL. +

+ ) : ( +
+

Endpoint URL *

+ onChange?.(e.target.value)} + /> +

+ Enter your full Azure DevOps Server base URL including the collection name. +

+
+ )} ); }; diff --git a/config-ui/src/plugins/register/azure/connection-fields/organization.tsx b/config-ui/src/plugins/register/azure/connection-fields/organization.tsx index 1e2972ca382..aae10c894e3 100644 --- a/config-ui/src/plugins/register/azure/connection-fields/organization.tsx +++ b/config-ui/src/plugins/register/azure/connection-fields/organization.tsx @@ -23,7 +23,7 @@ import { Block, ExternalLink } from '@/components'; import { DOC_URL } from '@/release'; interface Props { - initialValue: OrganizationSettings; + initialValue?: OrganizationSettings | null; value: string; label?: string; setValue: (value: string) => void; @@ -38,11 +38,11 @@ export const ConnectionOrganization = ({ label, initialValue, value, setValue }: const [settings, setSettings] = useState({ scoped: false, organization: '' }); useEffect(() => { - const org = initialValue.organization || ''; + const org = initialValue?.organization || ''; setValue(org); - setSettings({ organization: initialValue.organization, scoped: org !== '' }); - }, [initialValue.organization]); + setSettings({ organization: initialValue?.organization ?? '', scoped: org !== '' }); + }, [initialValue?.organization]); const handleChange = (e: RadioChangeEvent) => { const scoped = e.target.value;