initial load of yaml working

pkg/container/docker_build.go

@@ -0,0 +1,108 @@
+package container
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/builder/dockerignore"
+	"github.com/docker/docker/client"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/fileutils"
+	"github.com/nektos/act/pkg/common"
+	log "github.com/sirupsen/logrus"
+)
+
+// NewDockerBuildExecutorInput the input for the NewDockerBuildExecutor function
+type NewDockerBuildExecutorInput struct {
+	DockerExecutorInput
+	ContextDir string
+	ImageTag   string
+}
+
+// NewDockerBuildExecutor function to create a run executor for the container
+func NewDockerBuildExecutor(input NewDockerBuildExecutorInput) common.Executor {
+	return func() error {
+		input.Logger.Infof("docker build -t %s %s", input.ImageTag, input.ContextDir)
+		if input.Dryrun {
+			return nil
+		}
+
+		cli, err := client.NewClientWithOpts(client.FromEnv)
+		if err != nil {
+			return err
+		}
+		cli.NegotiateAPIVersion(input.Ctx)
+
+		input.Logger.Debugf("Building image from '%v'", input.ContextDir)
+
+		tags := []string{input.ImageTag}
+		options := types.ImageBuildOptions{
+			Tags: tags,
+		}
+
+		buildContext, err := createBuildContext(input.ContextDir, "Dockerfile")
+		if err != nil {
+			return err
+		}
+
+		defer buildContext.Close()
+
+		input.Logger.Debugf("Creating image from context dir '%s' with tag '%s'", input.ContextDir, input.ImageTag)
+		resp, err := cli.ImageBuild(input.Ctx, buildContext, options)
+
+		err = input.logDockerResponse(resp.Body, err != nil)
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+
+}
+func createBuildContext(contextDir string, relDockerfile string) (io.ReadCloser, error) {
+	log.Debugf("Creating archive for build context dir '%s' with relative dockerfile '%s'", contextDir, relDockerfile)
+
+	// And canonicalize dockerfile name to a platform-independent one
+	relDockerfile = archive.CanonicalTarNameForPath(relDockerfile)
+
+	f, err := os.Open(filepath.Join(contextDir, ".dockerignore"))
+	if err != nil && !os.IsNotExist(err) {
+		return nil, err
+	}
+	defer f.Close()
+
+	var excludes []string
+	if err == nil {
+		excludes, err = dockerignore.ReadAll(f)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// If .dockerignore mentions .dockerignore or the Dockerfile
+	// then make sure we send both files over to the daemon
+	// because Dockerfile is, obviously, needed no matter what, and
+	// .dockerignore is needed to know if either one needs to be
+	// removed. The daemon will remove them for us, if needed, after it
+	// parses the Dockerfile. Ignore errors here, as they will have been
+	// caught by validateContextDirectory above.
+	var includes = []string{"."}
+	keepThem1, _ := fileutils.Matches(".dockerignore", excludes)
+	keepThem2, _ := fileutils.Matches(relDockerfile, excludes)
+	if keepThem1 || keepThem2 {
+		includes = append(includes, ".dockerignore", relDockerfile)
+	}
+
+	compression := archive.Uncompressed
+	buildCtx, err := archive.TarWithOptions(contextDir, &archive.TarOptions{
+		Compression:     compression,
+		ExcludePatterns: excludes,
+		IncludeFiles:    includes,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return buildCtx, nil
+}

pkg/container/docker_common.go

@@ -0,0 +1,115 @@
+package container
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"github.com/docker/docker/pkg/stdcopy"
+	"io"
+	"os"
+
+	"github.com/sirupsen/logrus"
+)
+
+// DockerExecutorInput common input params
+type DockerExecutorInput struct {
+	Ctx    context.Context
+	Logger *logrus.Entry
+	Dryrun bool
+}
+
+type dockerMessage struct {
+	ID          string `json:"id"`
+	Stream      string `json:"stream"`
+	Error       string `json:"error"`
+	ErrorDetail struct {
+		Message string
+	}
+	Status   string `json:"status"`
+	Progress string `json:"progress"`
+}
+
+func (i *DockerExecutorInput) logDockerOutput(dockerResponse io.Reader) {
+	w := i.Logger.Writer()
+	_, err := stdcopy.StdCopy(w, w, dockerResponse)
+	if err != nil {
+		i.Logger.Error(err)
+	}
+}
+
+func (i *DockerExecutorInput) streamDockerOutput(dockerResponse io.Reader) {
+	out := os.Stdout
+	go func() {
+		<-i.Ctx.Done()
+		fmt.Println()
+	}()
+
+	_, err := io.Copy(out, dockerResponse)
+	if err != nil {
+		i.Logger.Error(err)
+	}
+}
+
+func (i *DockerExecutorInput) writeLog(isError bool, format string, args ...interface{}) {
+	if i.Logger == nil {
+		return
+	}
+	if isError {
+		i.Logger.Errorf(format, args...)
+	} else {
+		i.Logger.Debugf(format, args...)
+	}
+
+}
+
+func (i *DockerExecutorInput) logDockerResponse(dockerResponse io.ReadCloser, isError bool) error {
+	if dockerResponse == nil {
+		return nil
+	}
+	defer dockerResponse.Close()
+
+	scanner := bufio.NewScanner(dockerResponse)
+	msg := dockerMessage{}
+
+	for scanner.Scan() {
+		line := scanner.Bytes()
+
+		msg.ID = ""
+		msg.Stream = ""
+		msg.Error = ""
+		msg.ErrorDetail.Message = ""
+		msg.Status = ""
+		msg.Progress = ""
+
+		if err := json.Unmarshal(line, &msg); err != nil {
+			i.writeLog(false, "Unable to unmarshal line [%s] ==> %v", string(line), err)
+			continue
+		}
+
+		if msg.Error != "" {
+			i.writeLog(isError, "%s", msg.Error)
+			return errors.New(msg.Error)
+		}
+
+		if msg.ErrorDetail.Message != "" {
+			i.writeLog(isError, "%s", msg.ErrorDetail.Message)
+			return errors.New(msg.Error)
+		}
+
+		if msg.Status != "" {
+			if msg.Progress != "" {
+				i.writeLog(isError, "%s :: %s :: %s\n", msg.Status, msg.ID, msg.Progress)
+			} else {
+				i.writeLog(isError, "%s :: %s\n", msg.Status, msg.ID)
+			}
+		} else if msg.Stream != "" {
+			i.writeLog(isError, msg.Stream)
+		} else {
+			i.writeLog(false, "Unable to handle line: %s", string(line))
+		}
+	}
+
+	return nil
+}

pkg/container/docker_images.go

@@ -0,0 +1,33 @@
+package container
+
+import (
+	"context"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/client"
+)
+
+// ImageExistsLocally returns a boolean indicating if an image with the
+// requested name (and tag) exist in the local docker image store
+func ImageExistsLocally(ctx context.Context, imageName string) (bool, error) {
+	cli, err := client.NewClientWithOpts(client.FromEnv)
+	if err != nil {
+		return false, err
+	}
+	cli.NegotiateAPIVersion(ctx)
+
+	filters := filters.NewArgs()
+	filters.Add("reference", imageName)
+
+	imageListOptions := types.ImageListOptions{
+		Filters: filters,
+	}
+
+	images, err := cli.ImageList(ctx, imageListOptions)
+	if err != nil {
+		return false, err
+	}
+
+	return len(images) > 0, nil
+}

pkg/container/docker_images_test.go

@@ -0,0 +1,45 @@
+package container
+
+import (
+	"context"
+	"io/ioutil"
+	"testing"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/client"
+	log "github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+)
+
+func init() {
+	log.SetLevel(log.DebugLevel)
+}
+
+func TestImageExistsLocally(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+	// to help make this test reliable and not flaky, we need to have
+	// an image that will exist, and onew that won't exist
+
+	exists, err := ImageExistsLocally(context.TODO(), "library/alpine:this-random-tag-will-never-exist")
+	assert.Nil(t, err)
+	assert.Equal(t, false, exists)
+
+	// pull an image
+	cli, err := client.NewClientWithOpts(client.FromEnv)
+	assert.Nil(t, err)
+	cli.NegotiateAPIVersion(context.TODO())
+
+	// Chose alpine latest because it's so small
+	// maybe we should build an image instead so that tests aren't reliable on dockerhub
+	reader, err := cli.ImagePull(context.TODO(), "alpine:latest", types.ImagePullOptions{})
+	assert.Nil(t, err)
+	defer reader.Close()
+	_, err = ioutil.ReadAll(reader)
+	assert.Nil(t, err)
+
+	exists, err = ImageExistsLocally(context.TODO(), "alpine:latest")
+	assert.Nil(t, err)
+	assert.Equal(t, true, exists)
+}

pkg/container/docker_pull.go

@@ -0,0 +1,56 @@
+package container
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/client"
+	"github.com/nektos/act/pkg/common"
+)
+
+// NewDockerPullExecutorInput the input for the NewDockerPullExecutor function
+type NewDockerPullExecutorInput struct {
+	DockerExecutorInput
+	Image string
+}
+
+// NewDockerPullExecutor function to create a run executor for the container
+func NewDockerPullExecutor(input NewDockerPullExecutorInput) common.Executor {
+	return func() error {
+		input.Logger.Infof("docker pull %v", input.Image)
+
+		if input.Dryrun {
+			return nil
+		}
+
+		imageRef := cleanImage(input.Image)
+		input.Logger.Debugf("pulling image '%v'", imageRef)
+
+		cli, err := client.NewClientWithOpts(client.FromEnv)
+		if err != nil {
+			return err
+		}
+		cli.NegotiateAPIVersion(input.Ctx)
+
+		reader, err := cli.ImagePull(input.Ctx, imageRef, types.ImagePullOptions{})
+		_ = input.logDockerResponse(reader, err != nil)
+		if err != nil {
+			return err
+		}
+		return nil
+
+	}
+
+}
+
+func cleanImage(image string) string {
+	imageParts := len(strings.Split(image, "/"))
+	if imageParts == 1 {
+		image = fmt.Sprintf("docker.io/library/%s", image)
+	} else if imageParts == 2 {
+		image = fmt.Sprintf("docker.io/%s", image)
+	}
+
+	return image
+}

pkg/container/docker_pull_test.go

@@ -0,0 +1,29 @@
+package container
+
+import (
+	"testing"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+)
+
+func init() {
+	log.SetLevel(log.DebugLevel)
+}
+
+func TestCleanImage(t *testing.T) {
+	tables := []struct {
+		imageIn  string
+		imageOut string
+	}{
+		{"myhost.com/foo/bar", "myhost.com/foo/bar"},
+		{"ubuntu", "docker.io/library/ubuntu"},
+		{"ubuntu:18.04", "docker.io/library/ubuntu:18.04"},
+		{"cibuilds/hugo:0.53", "docker.io/cibuilds/hugo:0.53"},
+	}
+
+	for _, table := range tables {
+		imageOut := cleanImage(table.imageIn)
+		assert.Equal(t, table.imageOut, imageOut)
+	}
+}

pkg/container/docker_run.go

@@ -0,0 +1,211 @@
+package container
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/client"
+	"github.com/nektos/act/pkg/common"
+	"golang.org/x/crypto/ssh/terminal"
+)
+
+// NewDockerRunExecutorInput the input for the NewDockerRunExecutor function
+type NewDockerRunExecutorInput struct {
+	DockerExecutorInput
+	Image           string
+	Entrypoint      []string
+	Cmd             []string
+	WorkingDir      string
+	Env             []string
+	Binds           []string
+	Content         map[string]io.Reader
+	Volumes         []string
+	Name            string
+	ReuseContainers bool
+}
+
+// NewDockerRunExecutor function to create a run executor for the container
+func NewDockerRunExecutor(input NewDockerRunExecutorInput) common.Executor {
+	return func() error {
+
+		input.Logger.Infof("docker run image=%s entrypoint=%+q cmd=%+q", input.Image, input.Entrypoint, input.Cmd)
+		if input.Dryrun {
+			return nil
+		}
+
+		cli, err := client.NewClientWithOpts(client.FromEnv)
+		if err != nil {
+			return err
+		}
+		cli.NegotiateAPIVersion(input.Ctx)
+
+		// check if container exists
+		containerID, err := findContainer(input, cli, input.Name)
+		if err != nil {
+			return err
+		}
+
+		// if we have an old container and we aren't reusing, remove it!
+		if !input.ReuseContainers && containerID != "" {
+			input.Logger.Debugf("Found existing container for %s...removing", input.Name)
+			removeContainer(input, cli, containerID)
+			containerID = ""
+		}
+
+		// create a new container if we don't have one to reuse
+		if containerID == "" {
+			containerID, err = createContainer(input, cli)
+			if err != nil {
+				return err
+			}
+		}
+
+		// be sure to cleanup container if we aren't reusing
+		if !input.ReuseContainers {
+			defer removeContainer(input, cli, containerID)
+		}
+
+		executor := common.NewPipelineExecutor(
+			func() error {
+				return copyContentToContainer(input, cli, containerID)
+			}, func() error {
+				return attachContainer(input, cli, containerID)
+			}, func() error {
+				return startContainer(input, cli, containerID)
+			}, func() error {
+				return waitContainer(input, cli, containerID)
+			},
+		)
+		return executor()
+	}
+
+}
+
+func createContainer(input NewDockerRunExecutorInput, cli *client.Client) (string, error) {
+	isTerminal := terminal.IsTerminal(int(os.Stdout.Fd()))
+
+	config := &container.Config{
+		Image:      input.Image,
+		Cmd:        input.Cmd,
+		Entrypoint: input.Entrypoint,
+		WorkingDir: input.WorkingDir,
+		Env:        input.Env,
+		Tty:        isTerminal,
+	}
+
+	if len(input.Volumes) > 0 {
+		config.Volumes = make(map[string]struct{})
+		for _, vol := range input.Volumes {
+			config.Volumes[vol] = struct{}{}
+		}
+	}
+
+	resp, err := cli.ContainerCreate(input.Ctx, config, &container.HostConfig{
+		Binds: input.Binds,
+	}, nil, input.Name)
+	if err != nil {
+		return "", err
+	}
+	input.Logger.Debugf("Created container name=%s id=%v from image %v", input.Name, resp.ID, input.Image)
+	input.Logger.Debugf("ENV ==> %v", input.Env)
+
+	return resp.ID, nil
+}
+
+func findContainer(input NewDockerRunExecutorInput, cli *client.Client, containerName string) (string, error) {
+	containers, err := cli.ContainerList(input.Ctx, types.ContainerListOptions{
+		All: true,
+	})
+	if err != nil {
+		return "", err
+	}
+
+	for _, container := range containers {
+		for _, name := range container.Names {
+			if name[1:] == containerName {
+				return container.ID, nil
+			}
+		}
+	}
+
+	return "", nil
+}
+
+func removeContainer(input NewDockerRunExecutorInput, cli *client.Client, containerID string) {
+	err := cli.ContainerRemove(context.Background(), containerID, types.ContainerRemoveOptions{
+		RemoveVolumes: true,
+		Force:         true,
+	})
+	if err != nil {
+		input.Logger.Errorf("%v", err)
+	}
+
+	input.Logger.Debugf("Removed container: %v", containerID)
+}
+
+func copyContentToContainer(input NewDockerRunExecutorInput, cli *client.Client, containerID string) error {
+	for dstPath, srcReader := range input.Content {
+		input.Logger.Debugf("Extracting content to '%s'", dstPath)
+		err := cli.CopyToContainer(input.Ctx, containerID, dstPath, srcReader, types.CopyToContainerOptions{})
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func attachContainer(input NewDockerRunExecutorInput, cli *client.Client, containerID string) error {
+	out, err := cli.ContainerAttach(input.Ctx, containerID, types.ContainerAttachOptions{
+		Stream: true,
+		Stdout: true,
+		Stderr: true,
+	})
+	if err != nil {
+		return err
+	}
+	isTerminal := terminal.IsTerminal(int(os.Stdout.Fd()))
+	if !isTerminal || os.Getenv("NORAW") != "" {
+		go input.logDockerOutput(out.Reader)
+	} else {
+		go input.streamDockerOutput(out.Reader)
+	}
+	return nil
+}
+
+func startContainer(input NewDockerRunExecutorInput, cli *client.Client, containerID string) error {
+	input.Logger.Debugf("STARTING image=%s entrypoint=%s cmd=%v", input.Image, input.Entrypoint, input.Cmd)
+
+	if err := cli.ContainerStart(input.Ctx, containerID, types.ContainerStartOptions{}); err != nil {
+		return err
+	}
+
+	input.Logger.Debugf("Started container: %v", containerID)
+	return nil
+}
+
+func waitContainer(input NewDockerRunExecutorInput, cli *client.Client, containerID string) error {
+	statusCh, errCh := cli.ContainerWait(input.Ctx, containerID, container.WaitConditionNotRunning)
+	var statusCode int64
+	select {
+	case err := <-errCh:
+		if err != nil {
+			return err
+		}
+	case status := <-statusCh:
+		statusCode = status.StatusCode
+	}
+
+	input.Logger.Debugf("Return status: %v", statusCode)
+
+	if statusCode == 0 {
+		return nil
+	} else if statusCode == 78 {
+		return fmt.Errorf("exit with `NEUTRAL`: 78")
+	}
+
+	return fmt.Errorf("exit with `FAILURE`: %v", statusCode)
+}

pkg/container/docker_run_test.go

@@ -0,0 +1,56 @@
+package container
+
+import (
+	"bytes"
+	"context"
+	"io/ioutil"
+	"testing"
+
+	"github.com/nektos/act/pkg/common"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+)
+
+type rawFormatter struct{}
+
+func (f *rawFormatter) Format(entry *logrus.Entry) ([]byte, error) {
+	return []byte(entry.Message), nil
+}
+
+func TestNewDockerRunExecutor(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping slower test")
+	}
+
+	noopLogger := logrus.New()
+	noopLogger.SetOutput(ioutil.Discard)
+
+	buf := &bytes.Buffer{}
+	logger := logrus.New()
+	logger.SetOutput(buf)
+	logger.SetFormatter(&rawFormatter{})
+
+	runner := NewDockerRunExecutor(NewDockerRunExecutorInput{
+		DockerExecutorInput: DockerExecutorInput{
+			Ctx:    context.TODO(),
+			Logger: logrus.NewEntry(logger),
+		},
+		Image: "hello-world",
+	})
+
+	puller := NewDockerPullExecutor(NewDockerPullExecutorInput{
+		DockerExecutorInput: DockerExecutorInput{
+			Ctx:    context.TODO(),
+			Logger: logrus.NewEntry(noopLogger),
+		},
+		Image: "hello-world",
+	})
+
+	pipeline := common.NewPipelineExecutor(puller, runner)
+	err := pipeline()
+	assert.NoError(t, err)
+
+	expected := `docker run image=hello-world entrypoint=[] cmd=[]Hello from Docker!`
+	actual := buf.String()
+	assert.Equal(t, expected, actual[:len(expected)])
+}