From 2af7ea19412c1eef8dabbb9a8f7db4567b5f27a6 Mon Sep 17 00:00:00 2001 From: Ivan Krutov Date: Thu, 14 Dec 2017 11:40:21 +0300 Subject: [PATCH] Added private registry support (fixes #98) --- Gopkg.lock | 10 ++++- cmd/selenoid.go | 12 +++--- docs/selenoid-commands.adoc | 5 +++ selenoid/base.go | 6 ++- selenoid/docker.go | 86 ++++++++++++++++++++++++++++++------- selenoid/docker_test.go | 31 +++++++++++-- 6 files changed, 119 insertions(+), 31 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 83db6a0..6744329 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -33,7 +33,7 @@ [[projects]] name = "github.com/docker/docker" - packages = ["api/types","api/types/blkiodev","api/types/container","api/types/events","api/types/filters","api/types/mount","api/types/network","api/types/reference","api/types/registry","api/types/strslice","api/types/swarm","api/types/time","api/types/versions","api/types/volume","client","pkg/tlsconfig"] + packages = ["api/types","api/types/blkiodev","api/types/container","api/types/events","api/types/filters","api/types/mount","api/types/network","api/types/reference","api/types/registry","api/types/strslice","api/types/swarm","api/types/time","api/types/versions","api/types/volume","cliconfig","cliconfig/configfile","client","pkg/homedir","pkg/tlsconfig"] revision = "092cba3727bb9b4a2f0e922cd6c0f93ea270e363" version = "v1.13.1" @@ -124,6 +124,12 @@ packages = ["."] revision = "4fdf99ab29366514c69ccccddab5dc58b8d84062" +[[projects]] + name = "github.com/opencontainers/runc" + packages = ["libcontainer/user"] + revision = "baf6536d6259209c3edfa2b22237af82942d3dfa" + version = "v0.1.1" + [[projects]] name = "github.com/pkg/errors" packages = ["."] @@ -180,6 +186,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "7b0521be4535e9a549d79291a79059c268141d15f38ee991e12c9c27123b4f26" + inputs-digest = "c2e2a974e49c38adfb19cfe42fb7571235ef38b6c28d52aa84418d8b7d4f9bbe" solver-name = "gps-cdcl" solver-version = 1 diff --git a/cmd/selenoid.go b/cmd/selenoid.go index 7efa321..d39f621 100644 --- a/cmd/selenoid.go +++ b/cmd/selenoid.go @@ -8,11 +8,6 @@ import ( "runtime" ) -const ( - registryUrl = "https://registry.hub.docker.com" - defaultBrowsersJsonURL = "https://raw.githubusercontent.com/aerokube/cm/master/browsers.json" -) - var ( lastVersions int tmpfs int @@ -111,9 +106,12 @@ func initFlags() { selenoidConfigureCmd, selenoidStartCmd, selenoidUpdateCmd, + selenoidDownloadUICmd, + selenoidStartUICmd, + selenoidUpdateUICmd, } { c.Flags().StringVarP(&version, "version", "v", selenoid.Latest, "desired version; default is latest release") - c.Flags().StringVarP(®istry, "registry", "r", registryUrl, "Docker registry to use") + c.Flags().StringVarP(®istry, "registry", "r", selenoid.DefaultRegistryUrl, "Docker registry to use") } for _, c := range []*cobra.Command{ selenoidConfigureCmd, @@ -122,7 +120,7 @@ func initFlags() { } { c.Flags().StringVarP(&browsers, "browsers", "b", "", "comma separated list of browser names to process") c.Flags().StringVarP(&browserEnv, "browser-env", "w", "", "override container or driver environment variables (e.g. \"KEY1=value1 KEY2=value2\")") - c.Flags().StringVarP(&browsersJSONUrl, "browsers-json", "j", defaultBrowsersJsonURL, "browsers JSON data URL (in most cases never need to be set manually)") + c.Flags().StringVarP(&browsersJSONUrl, "browsers-json", "j", selenoid.DefaultBrowsersJsonURL, "browsers JSON data URL (in most cases never need to be set manually)") c.Flags().BoolVarP(&skipDownload, "no-download", "n", false, "only output config file without downloading images or drivers") c.Flags().IntVarP(&lastVersions, "last-versions", "l", 2, "process only last N versions (Docker only)") c.Flags().IntVarP(&tmpfs, "tmpfs", "t", 0, "add tmpfs volume sized in megabytes (Docker only)") diff --git a/docs/selenoid-commands.adoc b/docs/selenoid-commands.adoc index bb045f4..af01b07 100644 --- a/docs/selenoid-commands.adoc +++ b/docs/selenoid-commands.adoc @@ -69,6 +69,11 @@ To override Selenoid startup arguments sessions add `--args` flag: $ ./cm selenoid start --args "-limit 10" + +To download images from private registry - log in with `docker login` command and add `--registry` flag: + + $ docker login my-registry.example.com # Specify user name and password + $ ./cm selenoid start --registry https://my-registry.example.com ++ An alternative to downloading `cm` manually is using Docker container: # docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v ${HOME}:/root -e OVERRIDE_HOME=${HOME} aerokube/cm:latest-release selenoid start diff --git a/selenoid/base.go b/selenoid/base.go index 180a3ac..e307d6e 100644 --- a/selenoid/base.go +++ b/selenoid/base.go @@ -114,8 +114,10 @@ type PortAware struct { } const ( - SelenoidDefaultPort = 4444 - SelenoidUIDefaultPort = 8080 + SelenoidDefaultPort = 4444 + SelenoidUIDefaultPort = 8080 + DefaultRegistryUrl = "https://registry.hub.docker.com" + DefaultBrowsersJsonURL = "https://raw.githubusercontent.com/aerokube/cm/master/browsers.json" ) func getHomeDir() string { diff --git a/selenoid/docker.go b/selenoid/docker.go index 04fddbb..839da08 100644 --- a/selenoid/docker.go +++ b/selenoid/docker.go @@ -21,6 +21,7 @@ import ( "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/network" "github.com/docker/docker/api/types/strslice" + authconfig "github.com/docker/docker/cliconfig" "github.com/docker/docker/client" "github.com/docker/go-connections/nat" "github.com/heroku/docker-registry-client/registry" @@ -31,8 +32,10 @@ import ( "regexp" "runtime" + "encoding/base64" "github.com/aerokube/cm/render/rewriter" "github.com/fatih/color" + "net/url" . "vbom.ml/util/sortorder" ) @@ -61,13 +64,15 @@ type DockerConfigurator struct { EnvAware BrowserEnvAware PortAware - LastVersions int - Pull bool - RegistryUrl string - Tmpfs int - VNC bool - docker *client.Client - reg *registry.Registry + LastVersions int + Pull bool + RegistryUrl string + Tmpfs int + VNC bool + docker *client.Client + reg *registry.Registry + authConfig *types.AuthConfig + registryHostname string } func NewDockerConfigurator(config *LifecycleConfig) (*DockerConfigurator, error) { @@ -94,6 +99,12 @@ func NewDockerConfigurator(config *LifecycleConfig) (*DockerConfigurator, error) if err != nil { return nil, fmt.Errorf("new configurator: %v", err) } + authConfig, err := c.initAuthConfig() + if err != nil { + + } else { + c.authConfig = authConfig + } err = c.initRegistryClient() if err != nil { return nil, fmt.Errorf("new configurator: %v", err) @@ -110,12 +121,38 @@ func (c *DockerConfigurator) initDockerClient() error { return nil } +func (c *DockerConfigurator) initAuthConfig() (*types.AuthConfig, error) { + configFile, err := authconfig.Load("") + if err != nil { + return nil, err + } + u, err := url.Parse(c.RegistryUrl) + if err != nil { + return nil, err + } + + registryHostname := u.Hostname() + if c.RegistryUrl != DefaultRegistryUrl { + c.registryHostname = registryHostname + } + if cfg, ok := configFile.AuthConfigs[registryHostname]; ok { + c.Titlef(`Loaded authentication data for "%s"`, registryHostname) + return &cfg, nil + } + + return nil, nil +} + func (c *DockerConfigurator) initRegistryClient() error { url := strings.TrimSuffix(c.RegistryUrl, "/") + username, password := "", "" + if c.authConfig != nil { + username, password = c.authConfig.Username, c.authConfig.Password + } reg := ®istry.Registry{ URL: url, Client: &http.Client{ - Transport: registry.WrapTransport(http.DefaultTransport, url, "", ""), + Transport: registry.WrapTransport(http.DefaultTransport, url, username, password), }, Logf: func(format string, args ...interface{}) { c.Tracef(format, args...) @@ -223,9 +260,9 @@ func (c *DockerConfigurator) downloadImpl(imageName string, version string, erro version = *latestVersion } } - ref := imageName + ref := c.getFullyQualifiedImageRef(imageName) if version != Latest { - ref = fmt.Sprintf("%s:%s", ref, version) + ref = imageWithTag(ref, version) } if !c.pullImage(context.Background(), ref) { return "", errors.New(errorMessage) @@ -280,14 +317,15 @@ func (c *DockerConfigurator) createConfig() SelenoidConfig { tags := c.fetchImageTags(image) image, tags = c.preProcessImageTags(image, browserName, tags) pulledTags := tags + fullyQualifiedImage := c.getFullyQualifiedImageRef(image) if c.DownloadNeeded { - pulledTags = c.pullImages(image, tags) + pulledTags = c.pullImages(fullyQualifiedImage, tags) } else if c.LastVersions > 0 && c.LastVersions <= len(tags) { pulledTags = tags[:c.LastVersions] } if len(pulledTags) > 0 { - browsers[browserName] = c.createVersions(browserName, image, pulledTags) + browsers[browserName] = c.createVersions(browserName, fullyQualifiedImage, pulledTags) } } if c.DownloadNeeded { @@ -299,8 +337,8 @@ func (c *DockerConfigurator) createConfig() SelenoidConfig { func (c *DockerConfigurator) getSupportedBrowsers() map[string]string { return map[string]string{ "firefox": "selenoid/firefox", - "chrome": "selenoid/chrome", - "opera": "selenoid/opera", + "chrome": "selenoid/chrome", + "opera": "selenoid/opera", } } @@ -379,7 +417,14 @@ loop: func (c *DockerConfigurator) pullVideoRecorderImage() { c.Titlef("Pulling video recorder image...") - c.pullImage(context.Background(), videoRecorderImage) + c.pullImage(context.Background(), c.getFullyQualifiedImageRef(videoRecorderImage)) +} + +func (c *DockerConfigurator) getFullyQualifiedImageRef(ref string) string { + if c.registryHostname != "" { + return fmt.Sprintf("%s/%s", c.registryHostname, ref) + } + return ref } func (c *DockerConfigurator) preProcessImageTags(image string, browserName string, tags []string) (string, []string) { @@ -431,7 +476,16 @@ type JSONProgress struct { func (c *DockerConfigurator) pullImage(ctx context.Context, ref string) bool { c.Pointf("Pulling image %v", color.BlueString(ref)) - resp, err := c.docker.ImagePull(ctx, ref, types.ImagePullOptions{}) + pullOptions := types.ImagePullOptions{} + if c.authConfig != nil { + buf, err := json.Marshal(c.authConfig) + if err != nil { + c.Errorf("Failed to prepare registry authentication config: %v", err) + } else { + pullOptions.RegistryAuth = base64.URLEncoding.EncodeToString(buf) + } + } + resp, err := c.docker.ImagePull(ctx, ref, pullOptions) if err != nil { c.Errorf(`Failed to pull image "%s": %v`, ref, err) return false diff --git a/selenoid/docker_test.go b/selenoid/docker_test.go index 51d5c06..45a31d0 100644 --- a/selenoid/docker_test.go +++ b/selenoid/docker_test.go @@ -60,6 +60,21 @@ func mux() http.Handler { w.WriteHeader(http.StatusOK) }, )) + + mux.HandleFunc("/v2/aerokube/selenoid/tags/list", http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("Content-Type", "application/json") + fmt.Fprintln(w, `{"name":"selenoid", "tags": ["1.4.0", "1.4.1"]}`) + }, + )) + + mux.HandleFunc("/v2/aerokube/selenoid-ui/tags/list", http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("Content-Type", "application/json") + fmt.Fprintln(w, `{"name":"selenoid-ui", "tags": ["1.5.2"]}`) + }, + )) + mux.HandleFunc("/v2/selenoid/firefox/tags/list", http.HandlerFunc( func(w http.ResponseWriter, r *http.Request) { w.Header().Add("Content-Type", "application/json") @@ -249,6 +264,7 @@ func testConfigure(t *testing.T, download bool) { c, err := NewDockerConfigurator(&lcConfig) AssertThat(t, err, Is{nil}) defer c.Close() + c.registryHostname = "" AssertThat(t, c.IsConfigured(), Is{false}) cfgPointer, err := (*c).Configure() AssertThat(t, err, Is{nil}) @@ -290,12 +306,17 @@ func testConfigure(t *testing.T, download bool) { AssertThat(t, operaVersions.Default, EqualTo{"44.0"}) correctOperaBrowsers := make(map[string]*config.Browser) - correctOperaBrowsers["2.1.1"] = &config.Browser{ - Image: "selenoid/opera:44.0", + correctOperaBrowsers["44.0"] = &config.Browser{ + Image: "selenoid/vnc:opera_44.0", Port: "4444", + Path: "/", Tmpfs: tmpfsMap, Env: []string{testEnv}, } + AssertThat(t, operaVersions, EqualTo{config.Versions{ + Default: "44.0", + Versions: correctOperaBrowsers, + }}) }) } @@ -337,8 +358,9 @@ func TestDownload(t *testing.T) { Quiet: true, Version: Latest, }) - AssertThat(t, c.IsDownloaded(), Is{true}) AssertThat(t, err, Is{nil}) + c.registryHostname = "" + AssertThat(t, c.IsDownloaded(), Is{true}) ref, err := c.Download() AssertThat(t, ref, Not{nil}) AssertThat(t, err, Is{nil}) @@ -354,8 +376,9 @@ func TestDownloadUI(t *testing.T) { Version: Latest, }) setImageName(selenoidUIImage) - AssertThat(t, c.IsUIDownloaded(), Is{true}) AssertThat(t, err, Is{nil}) + c.registryHostname = "" + AssertThat(t, c.IsUIDownloaded(), Is{true}) ref, err := c.DownloadUI() AssertThat(t, ref, Not{nil}) AssertThat(t, err, Is{nil})