Merge branch 'master' into upgrade-spark-operator-chart-1.1.15

boilerplate/flyte/golang_support_tools/go.mod

@@ -1,13 +1,191 @@
 module github.com/flyteorg/boilerplate
 
-go 1.16
+go 1.17
 
 require (
 	github.com/alvaroloes/enumer v1.1.2
 	github.com/flyteorg/flytestdlib v0.4.7
 	github.com/golangci/golangci-lint v1.38.0
+	github.com/pseudomuto/protoc-gen-doc v1.4.1
 	github.com/vektra/mockery v0.0.0-20181123154057-e78b021dcbb5
-	github.com/pseudomuto/protoc-gen-doc v0.1.1 // indirect
+)
+
+require (
+	4d63.com/gochecknoglobals v0.0.0-20201008074935-acfc0b28355a // indirect
+	cloud.google.com/go v0.75.0 // indirect
+	cloud.google.com/go/storage v1.12.0 // indirect
+	github.com/Azure/azure-sdk-for-go v51.0.0+incompatible // indirect
+	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
+	github.com/Azure/go-autorest/autorest v0.11.17 // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.9.10 // indirect
+	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
+	github.com/Azure/go-autorest/logger v0.2.0 // indirect
+	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/BurntSushi/toml v0.3.1 // indirect
+	github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 // indirect
+	github.com/Masterminds/semver v1.5.0 // indirect
+	github.com/Masterminds/sprig v2.15.0+incompatible // indirect
+	github.com/OpenPeeDeeP/depguard v1.0.1 // indirect
+	github.com/alexkohler/prealloc v1.0.0 // indirect
+	github.com/aokoli/goutils v1.0.1 // indirect
+	github.com/ashanbrown/forbidigo v1.1.0 // indirect
+	github.com/ashanbrown/makezero v0.0.0-20201205152432-7b7cdbb3025a // indirect
+	github.com/aws/aws-sdk-go v1.37.1 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bkielbasa/cyclop v1.2.0 // indirect
+	github.com/bombsimon/wsl/v3 v3.2.0 // indirect
+	github.com/cespare/xxhash v1.1.0 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/charithe/durationcheck v0.0.6 // indirect
+	github.com/coocood/freecache v1.1.1 // indirect
+	github.com/daixiang0/gci v0.2.8 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/denis-tingajkin/go-header v0.4.2 // indirect
+	github.com/envoyproxy/protoc-gen-validate v0.3.0-java // indirect
+	github.com/ernesto-jimenez/gogen v0.0.0-20180125220232-d7d4131e6607 // indirect
+	github.com/esimonov/ifshort v1.0.1 // indirect
+	github.com/fatih/color v1.10.0 // indirect
+	github.com/fatih/structtag v1.2.0 // indirect
+	github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect
+	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/fzipp/gocyclo v0.3.1 // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/go-critic/go-critic v0.5.4 // indirect
+	github.com/go-logr/logr v0.4.0 // indirect
+	github.com/go-toolsmith/astcast v1.0.0 // indirect
+	github.com/go-toolsmith/astcopy v1.0.0 // indirect
+	github.com/go-toolsmith/astequal v1.0.0 // indirect
+	github.com/go-toolsmith/astfmt v1.0.0 // indirect
+	github.com/go-toolsmith/astp v1.0.0 // indirect
+	github.com/go-toolsmith/strparse v1.0.0 // indirect
+	github.com/go-toolsmith/typep v1.0.2 // indirect
+	github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b // indirect
+	github.com/gobwas/glob v0.2.3 // indirect
+	github.com/gofrs/flock v0.8.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/protobuf v1.4.3 // indirect
+	github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 // indirect
+	github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a // indirect
+	github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 // indirect
+	github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a // indirect
+	github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 // indirect
+	github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca // indirect
+	github.com/golangci/misspell v0.3.5 // indirect
+	github.com/golangci/revgrep v0.0.0-20210208091834-cd28932614b5 // indirect
+	github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 // indirect
+	github.com/google/go-cmp v0.5.4 // indirect
+	github.com/google/uuid v1.1.2 // indirect
+	github.com/googleapis/gax-go/v2 v2.0.5 // indirect
+	github.com/gordonklaus/ineffassign v0.0.0-20210225214923-2e10b2664254 // indirect
+	github.com/gostaticanalysis/analysisutil v0.4.1 // indirect
+	github.com/gostaticanalysis/comment v1.4.1 // indirect
+	github.com/gostaticanalysis/forcetypeassert v0.0.0-20200621232751-01d4955beaa5 // indirect
+	github.com/gostaticanalysis/nilerr v0.1.1 // indirect
+	github.com/graymeta/stow v0.2.7 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/huandu/xstrings v1.0.0 // indirect
+	github.com/imdario/mergo v0.3.5 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jgautheron/goconst v1.4.0 // indirect
+	github.com/jingyugao/rowserrcheck v0.0.0-20210130005344-c6a0c12dd98d // indirect
+	github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/jstemmer/go-junit-report v0.9.1 // indirect
+	github.com/julz/importas v0.0.0-20210226073942-60b4fa260dd0 // indirect
+	github.com/kisielk/errcheck v1.6.0 // indirect
+	github.com/kisielk/gotool v1.0.0 // indirect
+	github.com/kulti/thelper v0.4.0 // indirect
+	github.com/kunwardeep/paralleltest v1.0.2 // indirect
+	github.com/kyoh86/exportloopref v0.1.8 // indirect
+	github.com/magefile/mage v1.10.0 // indirect
+	github.com/magiconair/properties v1.8.4 // indirect
+	github.com/maratori/testpackage v1.0.1 // indirect
+	github.com/matoous/godox v0.0.0-20210227103229-6504466cf951 // indirect
+	github.com/mattn/go-colorable v0.1.8 // indirect
+	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/mattn/go-runewidth v0.0.7 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/mbilski/exhaustivestruct v1.2.0 // indirect
+	github.com/mgechev/dots v0.0.0-20190921121421-c36f7dcfbb81 // indirect
+	github.com/mgechev/revive v1.0.3 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/mapstructure v1.4.1 // indirect
+	github.com/moricho/tparallel v0.2.1 // indirect
+	github.com/mwitkow/go-proto-validators v0.0.0-20180403085117-0950a7990007 // indirect
+	github.com/nakabonne/nestif v0.3.0 // indirect
+	github.com/nbutton23/zxcvbn-go v0.0.0-20201221231540-e56b841a3c88 // indirect
+	github.com/ncw/swift v1.0.53 // indirect
+	github.com/nishanths/exhaustive v0.1.0 // indirect
+	github.com/nishanths/predeclared v0.2.1 // indirect
+	github.com/olekukonko/tablewriter v0.0.4 // indirect
+	github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1 // indirect
+	github.com/pelletier/go-toml v1.8.1 // indirect
+	github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/polyfloyd/go-errorlint v0.0.0-20201127212506-19bd8db6546f // indirect
+	github.com/prometheus/client_golang v1.9.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.15.0 // indirect
+	github.com/prometheus/procfs v0.3.0 // indirect
+	github.com/pseudomuto/protokit v0.2.0 // indirect
+	github.com/quasilyte/go-ruleguard v0.3.0 // indirect
+	github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95 // indirect
+	github.com/ryancurrah/gomodguard v1.2.0 // indirect
+	github.com/ryanrolds/sqlclosecheck v0.3.0 // indirect
+	github.com/sanposhiho/wastedassign v0.1.3 // indirect
+	github.com/satori/go.uuid v1.2.0 // indirect
+	github.com/securego/gosec/v2 v2.6.1 // indirect
+	github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c // indirect
+	github.com/sirupsen/logrus v1.8.0 // indirect
+	github.com/sonatard/noctx v0.0.1 // indirect
+	github.com/sourcegraph/go-diff v0.6.1 // indirect
+	github.com/spf13/afero v1.5.1 // indirect
+	github.com/spf13/cast v1.3.1 // indirect
+	github.com/spf13/cobra v1.1.3 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/spf13/viper v1.7.1 // indirect
+	github.com/ssgreg/nlreturn/v2 v2.1.0 // indirect
+	github.com/stretchr/objx v0.3.0 // indirect
+	github.com/stretchr/testify v1.7.0 // indirect
+	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/tdakkota/asciicheck v0.0.0-20200416200610-e657995f937b // indirect
+	github.com/tetafro/godot v1.4.4 // indirect
+	github.com/timakin/bodyclose v0.0.0-20200424151742-cb6215831a94 // indirect
+	github.com/tomarrell/wrapcheck v0.0.0-20201130113247-1683564d9756 // indirect
+	github.com/tommy-muehle/go-mnd/v2 v2.3.1 // indirect
+	github.com/ultraware/funlen v0.0.3 // indirect
+	github.com/ultraware/whitespace v0.0.4 // indirect
+	github.com/uudashr/gocognit v1.0.1 // indirect
+	go.opencensus.io v0.22.6 // indirect
+	golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect
+	golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect
+	golang.org/x/mod v0.4.1 // indirect
+	golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect
+	golang.org/x/oauth2 v0.0.0-20210126194326-f9ce19ea3013 // indirect
+	golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect
+	golang.org/x/text v0.3.5 // indirect
+	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
+	golang.org/x/tools v0.1.0 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/api v0.38.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506 // indirect
+	google.golang.org/grpc v1.35.0 // indirect
+	google.golang.org/protobuf v1.25.0 // indirect
+	gopkg.in/ini.v1 v1.62.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	honnef.co/go/tools v0.1.2 // indirect
+	k8s.io/apimachinery v0.20.2 // indirect
+	k8s.io/client-go v0.0.0-20210217172142-7279fc64d847 // indirect
+	k8s.io/klog/v2 v2.5.0 // indirect
+	mvdan.cc/gofumpt v0.1.0 // indirect
+	mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed // indirect
+	mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b // indirect
+	mvdan.cc/unparam v0.0.0-20210104141923-aac4ce9116a7 // indirect
 )
 
 replace github.com/vektra/mockery => github.com/enghabu/mockery v0.0.0-20191009061720-9d0c8670c2f0

boilerplate/flyte/golang_support_tools/tools.go

@@ -7,5 +7,5 @@ import (
 	_ "github.com/flyteorg/flytestdlib/cli/pflags"
 	_ "github.com/golangci/golangci-lint/cmd/golangci-lint"
 	_ "github.com/vektra/mockery/cmd/mockery"
-	- "github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc"
+	_ "github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc"
 )

boilerplate/flyte/golang_test_targets/go-gen.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -ex
+
+echo "Running go generate"
+go generate ./...
+
+# This section is used by GitHub workflow to ensure that the generation step was run
+if [ -n "$DELTA_CHECK" ]; then
+  DIRTY=$(git status --porcelain)
+  if [ -n "$DIRTY" ]; then
+    echo "FAILED: Go code updated without commiting generated code."
+    echo "Ensure make generate has run and all changes are committed."
+    DIFF=$(git diff)
+    echo "diff detected: $DIFF"
+    DIFF=$(git diff --name-only)
+    echo "files different: $DIFF"
+    exit 1
+  else
+    echo "SUCCESS: Generated code is up to date."
+  fi
+fi

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/config v1.0.0
 	github.com/aws/aws-sdk-go-v2/service/athena v1.0.0
 	github.com/coocood/freecache v1.1.1
-	github.com/flyteorg/flyteidl v0.21.11
+	github.com/flyteorg/flyteidl v0.21.23
 	github.com/flyteorg/flytestdlib v0.4.7
 	github.com/go-logr/zapr v0.4.0 // indirect
 	github.com/go-test/deep v1.0.7

go.sum

@@ -325,8 +325,8 @@ github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
-github.com/flyteorg/flyteidl v0.21.11 h1:oH9YPoR7scO9GFF/I8D0gCTOB+JP5HRK7b7cLUBRz90=
-github.com/flyteorg/flyteidl v0.21.11/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U=
+github.com/flyteorg/flyteidl v0.21.23 h1:hzGIFNOt3VooW/NdnaicXijn3EKjNKTz1kY+tlHkED4=
+github.com/flyteorg/flyteidl v0.21.23/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U=
 github.com/flyteorg/flytestdlib v0.3.13/go.mod h1:Tz8JCECAbX6VWGwFT6cmEQ+RJpZ/6L9pswu3fzWs220=
 github.com/flyteorg/flytestdlib v0.4.7 h1:SMPPXI3j/MjP7D2fqaR+lPQkTrqYS7xZbwsgJI2F8SU=
 github.com/flyteorg/flytestdlib v0.4.7/go.mod h1:fv1ar34LJLMTaf0tbfetisLykUlARi7rP+NQTUn6QQs=

go/tasks/config_load_test.go

@@ -87,6 +87,8 @@ func TestLoadConfig(t *testing.T) {
 		assert.NotNil(t, k8sConfig.DefaultSecurityContext)
 		assert.NotNil(t, k8sConfig.DefaultSecurityContext.AllowPrivilegeEscalation)
 		assert.False(t, *k8sConfig.DefaultSecurityContext.AllowPrivilegeEscalation)
+		assert.NotNil(t, k8sConfig.EnableHostNetworkingPod)
+		assert.True(t, *k8sConfig.EnableHostNetworkingPod)
 	})
 
 	t.Run("logs-config-test", func(t *testing.T) {

go/tasks/pluginmachinery/core/phase.go

@@ -4,8 +4,6 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/event"
-
 	"github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core"
 	structpb "github.com/golang/protobuf/ptypes/struct"
 )
@@ -69,6 +67,18 @@ func (p Phase) IsWaitingForResources() bool {
 	return p == PhaseWaitingForResources
 }
 
+type ExternalResource struct {
+	// A unique identifier for the external resource
+	ExternalID string
+	// A unique index for the external resource. Although the ID may change, this will remain the same
+	// throughout task event reports and retries.
+	Index uint32
+	// The nubmer of times this external resource has been attempted
+	RetryAttempt uint32
+	// Phase (if exists) associated with the external resource
+	Phase Phase
+}
+
 type TaskInfo struct {
 	// log information for the task execution
 	Logs []*core.TaskLog
@@ -77,8 +87,8 @@ type TaskInfo struct {
 	OccurredAt *time.Time
 	// Custom Event information that the plugin would like to expose to the front-end
 	CustomInfo *structpb.Struct
-	// Metadata around how a task was executed
-	Metadata *event.TaskExecutionMetadata
+	// A collection of information about external resources launched by this task
+	ExternalResources []*ExternalResource
 }
 
 func (t *TaskInfo) String() string {

go/tasks/pluginmachinery/flytek8s/config/config.go

@@ -139,6 +139,11 @@ type K8sPluginConfig struct {
 	// DefaultSecurityContext provides a default container security context that should be applied for the primary container launched and created by FlytePropeller. This may not be applicable to all plugins. For
 	//	// downstream plugins - i.e. TensorflowOperators may not support setting this, but Spark does.
 	DefaultSecurityContext *v1.SecurityContext `json:"default-security-context" pflag:"-,Optionally specify a default security context that should be applied to every container launched/created by FlytePropeller. This will not be applied to plugins that do not support it or to user supplied containers in pod tasks."`
+
+	// EnableHostNetworkingPod is a binary switch to enable `hostNetwork: true` for all pods launched by Flyte.
+	// Refer to - https://kubernetes.io/docs/concepts/policy/pod-security-policy/#host-namespaces.
+	// As a follow up, the default pod configurations will now be adjusted using podTemplates per namespace
+	EnableHostNetworkingPod *bool `json:"enable-host-networking-pod" pflag:"-,If true, will schedule all pods with hostNetwork: true."`
 }
 
 // FlyteCoPilotConfig specifies configuration for the Flyte CoPilot system. FlyteCoPilot, allows running flytekit-less containers

go/tasks/pluginmachinery/flytek8s/pod_helper.go

@@ -91,6 +91,9 @@ func UpdatePodWithInterruptibleFlag(taskExecutionMetadata pluginsCore.TaskExecut
 	if podSpec.SecurityContext == nil && config.GetK8sPluginConfig().DefaultPodSecurityContext != nil {
 		podSpec.SecurityContext = config.GetK8sPluginConfig().DefaultPodSecurityContext.DeepCopy()
 	}
+	if config.GetK8sPluginConfig().EnableHostNetworkingPod != nil {
+		podSpec.HostNetwork = *config.GetK8sPluginConfig().EnableHostNetworkingPod
+	}
 	ApplyInterruptibleNodeAffinity(isInterruptible, podSpec)
 }
 

go/tasks/pluginmachinery/flytek8s/pod_helper_test.go

@@ -494,6 +494,36 @@ func TestToK8sPod(t *testing.T) {
 		assert.NotNil(t, p.SecurityContext)
 		assert.Equal(t, *p.SecurityContext.RunAsGroup, v)
 	})
+
+	t.Run("enableHostNetwork", func(t *testing.T) {
+		enabled := true
+		assert.NoError(t, config.SetK8sPluginConfig(&config.K8sPluginConfig{
+			EnableHostNetworkingPod: &enabled,
+		}))
+		x := dummyExecContext(&v1.ResourceRequirements{})
+		p, err := ToK8sPodSpec(ctx, x)
+		assert.NoError(t, err)
+		assert.True(t, p.HostNetwork)
+	})
+
+	t.Run("explicitDisableHostNetwork", func(t *testing.T) {
+		enabled := false
+		assert.NoError(t, config.SetK8sPluginConfig(&config.K8sPluginConfig{
+			EnableHostNetworkingPod: &enabled,
+		}))
+		x := dummyExecContext(&v1.ResourceRequirements{})
+		p, err := ToK8sPodSpec(ctx, x)
+		assert.NoError(t, err)
+		assert.False(t, p.HostNetwork)
+	})
+
+	t.Run("skipSettingHostNetwork", func(t *testing.T) {
+		assert.NoError(t, config.SetK8sPluginConfig(&config.K8sPluginConfig{}))
+		x := dummyExecContext(&v1.ResourceRequirements{})
+		p, err := ToK8sPodSpec(ctx, x)
+		assert.NoError(t, err)
+		assert.False(t, p.HostNetwork)
+	})
 }
 
 func TestDemystifyPending(t *testing.T) {

go/tasks/pluginmachinery/webapi/example/plugin.go

@@ -4,8 +4,6 @@ import (
 	"context"
 	"time"
 
-	"github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/event"
-
 	idlCore "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core"
 
 	"github.com/flyteorg/flytestdlib/errors"
@@ -96,11 +94,9 @@ func (p Plugin) Status(ctx context.Context, tCtx webapi.StatusContext) (phase co
 			},
 		},
 		OccurredAt: &tNow,
-		Metadata: &event.TaskExecutionMetadata{
-			ExternalResources: []*event.ExternalResourceInfo{
-				{
-					ExternalId: "abc",
-				},
+		ExternalResources: []*core.ExternalResource{
+			{
+				ExternalID: "abc",
 			},
 		},
 	}), nil