Skip to content

add cli httpclient with support for proxy configuration #672

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
May 19, 2020
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion arduino/cores/packagemanager/download.go
Original file line number Diff line number Diff line change
@@ -19,7 +19,7 @@ import (
"fmt"

"github.com/arduino/arduino-cli/arduino/cores"
"go.bug.st/downloader"
"go.bug.st/downloader/v2"
semver "go.bug.st/relaxed-semver"
)

2 changes: 1 addition & 1 deletion arduino/libraries/librariesmanager/download.go
Original file line number Diff line number Diff line change
@@ -18,7 +18,7 @@ package librariesmanager
import (
"net/url"

"go.bug.st/downloader"
"go.bug.st/downloader/v2"
)

// LibraryIndexURL is the URL where to get library index.
2 changes: 1 addition & 1 deletion arduino/resources/helpers.go
Original file line number Diff line number Diff line change
@@ -20,7 +20,7 @@ import (
"os"

"github.com/arduino/go-paths-helper"
"go.bug.st/downloader"
"go.bug.st/downloader/v2"
)

// ArchivePath returns the path of the Archive of the specified DownloadResource relative
7 changes: 5 additions & 2 deletions arduino/resources/helpers_test.go
Original file line number Diff line number Diff line change
@@ -23,9 +23,10 @@ import (
"strings"
"testing"

"github.com/arduino/arduino-cli/httpclient"
"github.com/arduino/go-paths-helper"
"github.com/stretchr/testify/require"
"go.bug.st/downloader"
"go.bug.st/downloader/v2"
)

type EchoHandler struct{}
@@ -53,7 +54,9 @@ func TestDownloadApplyUserAgentHeaderUsingConfig(t *testing.T) {
URL: srv.URL,
}

d, err := r.Download(tmp, &downloader.Config{RequestHeaders: http.Header{"User-Agent": []string{goldUserAgentValue}}})
httpClient := httpclient.NewWithConfig(&httpclient.Config{UserAgent: goldUserAgentValue})

d, err := r.Download(tmp, &downloader.Config{HttpClient: *httpClient})
require.NoError(t, err)
err = d.Run()
require.NoError(t, err)
2 changes: 1 addition & 1 deletion arduino/resources/resources_test.go
Original file line number Diff line number Diff line change
@@ -22,7 +22,7 @@ import (

"github.com/arduino/go-paths-helper"
"github.com/stretchr/testify/require"
"go.bug.st/downloader"
"go.bug.st/downloader/v2"
)

func TestDownloadAndChecksums(t *testing.T) {
18 changes: 0 additions & 18 deletions cli/globals/globals.go
Original file line number Diff line number Diff line change
@@ -16,11 +16,8 @@
package globals

import (
"fmt"
"net/http"
"os"
"path/filepath"
"runtime"

"github.com/arduino/arduino-cli/version"
)
@@ -31,18 +28,3 @@ var (
// DefaultIndexURL is the default index url
DefaultIndexURL = "https://downloads.arduino.cc/packages/package_index.json"
)

// NewHTTPClientHeader returns the http.Header object that must be used by the clients inside the downloaders
// and adds the subComponent if specified
func NewHTTPClientHeader(subComponent string) http.Header {
if subComponent != "" {
subComponent = " " + subComponent
}
userAgentValue := fmt.Sprintf("%s/%s%s (%s; %s; %s) Commit:%s",
VersionInfo.Application,
VersionInfo.VersionString,
subComponent,
runtime.GOARCH, runtime.GOOS, runtime.Version(),
VersionInfo.Commit)
return http.Header{"User-Agent": []string{userAgentValue}}
}
11 changes: 8 additions & 3 deletions commands/board/list.go
Original file line number Diff line number Diff line change
@@ -23,8 +23,8 @@ import (
"regexp"
"sync"

"github.com/arduino/arduino-cli/cli/globals"
"github.com/arduino/arduino-cli/commands"
"github.com/arduino/arduino-cli/httpclient"
rpc "github.com/arduino/arduino-cli/rpc/commands"
"github.com/pkg/errors"
"github.com/segmentio/stats/v4"
@@ -51,12 +51,17 @@ func apiByVidPid(vid, pid string) ([]*rpc.BoardListItem, error) {
url := fmt.Sprintf("%s/%s/%s", vidPidURL, vid, pid)
retVal := []*rpc.BoardListItem{}
req, _ := http.NewRequest("GET", url, nil)
req.Header = globals.NewHTTPClientHeader("")
req.Header.Set("Content-Type", "application/json")

// TODO: use proxy if set

if res, err := http.DefaultClient.Do(req); err == nil {
httpClient, err := httpclient.New()

if err != nil {
return nil, errors.Wrap(err, "failed to initialize http client")
}

if res, err := httpClient.Do(req); err == nil {
if res.StatusCode >= 400 {
if res.StatusCode == 404 {
return nil, ErrNotFound
24 changes: 9 additions & 15 deletions commands/download.go
Original file line number Diff line number Diff line change
@@ -16,30 +16,24 @@
package commands

import (
"net/url"
"time"

"github.com/arduino/arduino-cli/cli/globals"
"github.com/arduino/arduino-cli/httpclient"
rpc "github.com/arduino/arduino-cli/rpc/commands"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/spf13/viper"
"go.bug.st/downloader"
"go.bug.st/downloader/v2"
)

// GetDownloaderConfig returns the downloader configuration based on
// current settings.
func GetDownloaderConfig() (*downloader.Config, error) {
res := &downloader.Config{
RequestHeaders: globals.NewHTTPClientHeader(viper.GetString("network.user_agent_ext")),

httpClient, err := httpclient.New()
if err != nil {
return nil, err
}
if viper.IsSet("network.proxy") {
proxy := viper.GetString("network.proxy")
if _, err := url.Parse(proxy); err != nil {
return nil, errors.New("Invalid network.proxy '" + proxy + "': " + err.Error())
}
res.ProxyURL = proxy
logrus.Infof("Using proxy %s", proxy)

res := &downloader.Config{
HttpClient: *httpClient,
}
return res, nil
}
2 changes: 1 addition & 1 deletion commands/instances.go
Original file line number Diff line number Diff line change
@@ -33,7 +33,7 @@ import (
paths "github.com/arduino/go-paths-helper"
"github.com/sirupsen/logrus"
"github.com/spf13/viper"
"go.bug.st/downloader"
"go.bug.st/downloader/v2"
)

// this map contains all the running Arduino Core Services instances
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
@@ -38,7 +38,7 @@ require (
github.com/spf13/viper v1.6.2
github.com/stretchr/testify v1.4.0
go.bug.st/cleanup v1.0.0
go.bug.st/downloader v1.2.0
go.bug.st/downloader/v2 v2.0.1
go.bug.st/relaxed-semver v0.0.0-20190922224835-391e10178d18
go.bug.st/serial v1.0.0
go.bug.st/serial.v1 v0.0.0-20180827123349-5f7892a7bb45 // indirect
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
@@ -205,8 +205,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
go.bug.st/cleanup v1.0.0 h1:XVj1HZxkBXeq3gMT7ijWUpHyIC1j8XAoNSyQ06CskgA=
go.bug.st/cleanup v1.0.0/go.mod h1:EqVmTg2IBk4znLbPD28xne3abjsJftMdqqJEjhn70bk=
go.bug.st/downloader v1.2.0 h1:YmXFTcTnm0v8WzAWHn2DyV46c/Izlc/gReXubc2oBho=
go.bug.st/downloader v1.2.0/go.mod h1:l+RPbNbrTB+MoAIp8nrZsP22nRPDy26XJZQqmm4gNT4=
go.bug.st/downloader/v2 v2.0.1 h1:F/ZgVDrUHo2tHIeOVbcl4rgGaMmQgAEjn5c8yi5t+Iw=
go.bug.st/downloader/v2 v2.0.1/go.mod h1:VZW2V1iGKV8rJL2ZEGIDzzBeKowYv34AedJz13RzVII=
go.bug.st/relaxed-semver v0.0.0-20190922224835-391e10178d18 h1:F1qxtaFuewctYc/SsHRn+Q7Dtwi+yJGPgVq8YLtQz98=
go.bug.st/relaxed-semver v0.0.0-20190922224835-391e10178d18/go.mod h1:Cx1VqMtEhE9pIkEyUj3LVVVPkv89dgW8aCKrRPDR/uE=
go.bug.st/serial v1.0.0 h1:ogEPzrllCsnG00EqKRjeYvPRsO7NJW6DqykzkdD6E/k=
40 changes: 40 additions & 0 deletions httpclient/httpclient.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
// This file is part of arduino-cli.
//
// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
//
// This software is released under the GNU General Public License version 3,
// which covers the main part of arduino-cli.
// The terms of this license can be found at:
// https://www.gnu.org/licenses/gpl-3.0.en.html
//
// You can be released from the requirements of the above licenses by purchasing
// a commercial license. Buying such a license is mandatory if you want to
// modify or otherwise use the software for commercial activities involving the
// Arduino software without disclosing the source code of your own applications.
// To purchase a commercial license, send an email to [email protected].

package httpclient

import (
"net/http"
)

// New returns a default http client for use in the cli API calls
func New() (*http.Client, error) {
config, err := DefaultConfig()

if err != nil {
return nil, err
}

return NewWithConfig(config), nil
}

// NewWithConfig creates a http client for use in the cli API calls with a given configuration
func NewWithConfig(config *Config) *http.Client {
transport := newHTTPClientTransport(config)

return &http.Client{
Transport: transport,
}
}
64 changes: 64 additions & 0 deletions httpclient/httpclient_config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
// This file is part of arduino-cli.
//
// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
//
// This software is released under the GNU General Public License version 3,
// which covers the main part of arduino-cli.
// The terms of this license can be found at:
// https://www.gnu.org/licenses/gpl-3.0.en.html
//
// You can be released from the requirements of the above licenses by purchasing
// a commercial license. Buying such a license is mandatory if you want to
// modify or otherwise use the software for commercial activities involving the
// Arduino software without disclosing the source code of your own applications.
// To purchase a commercial license, send an email to [email protected].

package httpclient

import (
"errors"
"fmt"
"net/url"
"runtime"

"github.com/arduino/arduino-cli/cli/globals"
"github.com/spf13/viper"
)

// Config is the configuration of the http client
type Config struct {
UserAgent string
Proxy *url.URL
}

// DefaultConfig returns the default http client config
func DefaultConfig() (*Config, error) {
var proxy *url.URL
var err error
if viper.IsSet("network.proxy") {
proxyConfig := viper.GetString("network.proxy")
if proxy, err = url.Parse(proxyConfig); err != nil {
return nil, errors.New("Invalid network.proxy '" + proxyConfig + "': " + err.Error())
}
}

return &Config{
UserAgent: UserAgent(),
Proxy: proxy,
}, nil
}

// UserAgent returns the user agent for the cli http client
func UserAgent() string {
subComponent := viper.GetString("network.user_agent_ext")
if subComponent != "" {
subComponent = " " + subComponent
}

return fmt.Sprintf("%s/%s%s (%s; %s; %s) Commit:%s",
globals.VersionInfo.Application,
globals.VersionInfo.VersionString,
subComponent,
runtime.GOARCH, runtime.GOOS, runtime.Version(),
globals.VersionInfo.Commit)
}
70 changes: 70 additions & 0 deletions httpclient/httpclient_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
// This file is part of arduino-cli.
//
// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
//
// This software is released under the GNU General Public License version 3,
// which covers the main part of arduino-cli.
// The terms of this license can be found at:
// https://www.gnu.org/licenses/gpl-3.0.en.html
//
// You can be released from the requirements of the above licenses by purchasing
// a commercial license. Buying such a license is mandatory if you want to
// modify or otherwise use the software for commercial activities involving the
// Arduino software without disclosing the source code of your own applications.
// To purchase a commercial license, send an email to [email protected].

package httpclient

import (
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"net/url"
"testing"

"github.com/stretchr/testify/require"
)

func TestUserAgentHeader(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, r.Header.Get("User-Agent"))
}))
defer ts.Close()

client := NewWithConfig(&Config{
UserAgent: "test-user-agent",
})

request, err := http.NewRequest("GET", ts.URL, nil)
require.NoError(t, err)

response, err := client.Do(request)
require.NoError(t, err)

b, err := ioutil.ReadAll(response.Body)
require.NoError(t, err)

require.Equal(t, "test-user-agent", string(b))
}

func TestProxy(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent)
}))
defer ts.Close()

proxyURL, err := url.Parse(ts.URL)
require.NoError(t, err)

client := NewWithConfig(&Config{
Proxy: proxyURL,
})

request, err := http.NewRequest("GET", "http://arduino.cc", nil)
require.NoError(t, err)

response, err := client.Do(request)
require.NoError(t, err)
require.Equal(t, http.StatusNoContent, response.StatusCode)
}
41 changes: 41 additions & 0 deletions httpclient/httpclient_transport.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
// This file is part of arduino-cli.
//
// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
//
// This software is released under the GNU General Public License version 3,
// which covers the main part of arduino-cli.
// The terms of this license can be found at:
// https://www.gnu.org/licenses/gpl-3.0.en.html
//
// You can be released from the requirements of the above licenses by purchasing
// a commercial license. Buying such a license is mandatory if you want to
// modify or otherwise use the software for commercial activities involving the
// Arduino software without disclosing the source code of your own applications.
// To purchase a commercial license, send an email to license@arduino.cc.

package httpclient

import "net/http"

type httpClientRoundTripper struct {
transport http.RoundTripper
config *Config
}

func newHTTPClientTransport(config *Config) http.RoundTripper {
proxy := http.ProxyURL(config.Proxy)

transport := &http.Transport{
Proxy: proxy,
}

return &httpClientRoundTripper{
transport: transport,
config: config,
}
}

func (h *httpClientRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
req.Header.Add("User-Agent", h.config.UserAgent)
return h.transport.RoundTrip(req)
}