feat: Host environment (#1293)

pkg/container/docker_run.go

@@ -84,7 +84,7 @@ type Container interface {
 }
 
 // NewContainer creates a reference to a container
-func NewContainer(input *NewContainerInput) Container {
+func NewContainer(input *NewContainerInput) ExecutionsEnvironment {
 	cr := new(containerReference)
 	cr.input = input
 	return cr
@@ -233,6 +233,7 @@ type containerReference struct {
 	input *NewContainerInput
 	UID   int
 	GID   int
+	LinuxContainerEnvironmentExtensions
 }
 
 func GetDockerClient(ctx context.Context) (cli client.APIClient, err error) {

pkg/container/docker_run_test.go

@@ -163,3 +163,6 @@ func TestDockerExecFailure(t *testing.T) {
 	conn.AssertExpectations(t)
 	client.AssertExpectations(t)
 }
+
+// Type assert containerReference implements ExecutionsEnvironment
+var _ ExecutionsEnvironment = &containerReference{}

pkg/container/executions_environment.go

@@ -0,0 +1,13 @@
+package container
+
+import "context"
+
+type ExecutionsEnvironment interface {
+	Container
+	ToContainerPath(string) string
+	GetActPath() string
+	GetPathVariableName() string
+	DefaultPathVariable() string
+	JoinPathVariable(...string) string
+	GetRunnerContext(ctx context.Context) map[string]interface{}
+}

pkg/container/file_collector.go

@@ -59,6 +59,29 @@ func (tc tarCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string,
 	return nil
 }
 
+type copyCollector struct {
+	DstDir string
+}
+
+func (cc *copyCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string, f io.Reader) error {
+	fdestpath := filepath.Join(cc.DstDir, fpath)
+	if err := os.MkdirAll(filepath.Dir(fdestpath), 0777); err != nil {
+		return err
+	}
+	if f == nil {
+		return os.Symlink(linkName, fdestpath)
+	}
+	df, err := os.OpenFile(fdestpath, os.O_CREATE|os.O_WRONLY, fi.Mode())
+	if err != nil {
+		return err
+	}
+	defer df.Close()
+	if _, err := io.Copy(df, f); err != nil {
+		return err
+	}
+	return nil
+}
+
 type fileCollector struct {
 	Ignorer   gitignore.Matcher
 	SrcPath   string

pkg/container/host_environment.go

@@ -0,0 +1,470 @@
+package container
+
+import (
+	"archive/tar"
+	"bufio"
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	"errors"
+
+	"github.com/go-git/go-billy/v5/helper/polyfill"
+	"github.com/go-git/go-billy/v5/osfs"
+	"github.com/go-git/go-git/v5/plumbing/format/gitignore"
+	"github.com/nektos/act/pkg/common"
+	"github.com/nektos/act/pkg/lookpath"
+	"golang.org/x/term"
+)
+
+type HostEnvironment struct {
+	Path      string
+	TmpDir    string
+	ToolCache string
+	Workdir   string
+	ActPath   string
+	CleanUp   func()
+	StdOut    io.Writer
+}
+
+func (e *HostEnvironment) Create(capAdd []string, capDrop []string) common.Executor {
+	return func(ctx context.Context) error {
+		return nil
+	}
+}
+
+func (e *HostEnvironment) Close() common.Executor {
+	return func(ctx context.Context) error {
+		return nil
+	}
+}
+
+func (e *HostEnvironment) Copy(destPath string, files ...*FileEntry) common.Executor {
+	return func(ctx context.Context) error {
+		for _, f := range files {
+			if err := os.MkdirAll(filepath.Dir(filepath.Join(destPath, f.Name)), 0777); err != nil {
+				return err
+			}
+			if err := os.WriteFile(filepath.Join(destPath, f.Name), []byte(f.Body), fs.FileMode(f.Mode)); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+}
+
+func (e *HostEnvironment) CopyDir(destPath string, srcPath string, useGitIgnore bool) common.Executor {
+	return func(ctx context.Context) error {
+		logger := common.Logger(ctx)
+		srcPrefix := filepath.Dir(srcPath)
+		if !strings.HasSuffix(srcPrefix, string(filepath.Separator)) {
+			srcPrefix += string(filepath.Separator)
+		}
+		logger.Debugf("Stripping prefix:%s src:%s", srcPrefix, srcPath)
+		var ignorer gitignore.Matcher
+		if useGitIgnore {
+			ps, err := gitignore.ReadPatterns(polyfill.New(osfs.New(srcPath)), nil)
+			if err != nil {
+				logger.Debugf("Error loading .gitignore: %v", err)
+			}
+
+			ignorer = gitignore.NewMatcher(ps)
+		}
+		fc := &fileCollector{
+			Fs:        &defaultFs{},
+			Ignorer:   ignorer,
+			SrcPath:   srcPath,
+			SrcPrefix: srcPrefix,
+			Handler: &copyCollector{
+				DstDir: destPath,
+			},
+		}
+		return filepath.Walk(srcPath, fc.collectFiles(ctx, []string{}))
+	}
+}
+
+func (e *HostEnvironment) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) {
+	buf := &bytes.Buffer{}
+	tw := tar.NewWriter(buf)
+	defer tw.Close()
+	srcPath = filepath.Clean(srcPath)
+	fi, err := os.Lstat(srcPath)
+	if err != nil {
+		return nil, err
+	}
+	tc := &tarCollector{
+		TarWriter: tw,
+	}
+	if fi.IsDir() {
+		srcPrefix := filepath.Dir(srcPath)
+		if !strings.HasSuffix(srcPrefix, string(filepath.Separator)) {
+			srcPrefix += string(filepath.Separator)
+		}
+		fc := &fileCollector{
+			Fs:        &defaultFs{},
+			SrcPath:   srcPath,
+			SrcPrefix: srcPrefix,
+			Handler:   tc,
+		}
+		err = filepath.Walk(srcPath, fc.collectFiles(ctx, []string{}))
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		var f io.ReadCloser
+		var linkname string
+		if fi.Mode()&fs.ModeSymlink != 0 {
+			linkname, err = os.Readlink(srcPath)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			f, err = os.Open(srcPath)
+			if err != nil {
+				return nil, err
+			}
+			defer f.Close()
+		}
+		err := tc.WriteFile(fi.Name(), fi, linkname, f)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return io.NopCloser(buf), nil
+}
+
+func (e *HostEnvironment) Pull(forcePull bool) common.Executor {
+	return func(ctx context.Context) error {
+		return nil
+	}
+}
+
+func (e *HostEnvironment) Start(attach bool) common.Executor {
+	return func(ctx context.Context) error {
+		return nil
+	}
+}
+
+type ptyWriter struct {
+	Out       io.Writer
+	AutoStop  bool
+	dirtyLine bool
+}
+
+func (w *ptyWriter) Write(buf []byte) (int, error) {
+	if w.AutoStop && len(buf) > 0 && buf[len(buf)-1] == 4 {
+		n, err := w.Out.Write(buf[:len(buf)-1])
+		if err != nil {
+			return n, err
+		}
+		if w.dirtyLine || len(buf) > 1 && buf[len(buf)-2] != '\n' {
+			_, _ = w.Out.Write([]byte("\n"))
+			return n, io.EOF
+		}
+		return n, io.EOF
+	}
+	w.dirtyLine = strings.LastIndex(string(buf), "\n") < len(buf)-1
+	return w.Out.Write(buf)
+}
+
+type localEnv struct {
+	env map[string]string
+}
+
+func (l *localEnv) Getenv(name string) string {
+	if runtime.GOOS == "windows" {
+		for k, v := range l.env {
+			if strings.EqualFold(name, k) {
+				return v
+			}
+		}
+		return ""
+	}
+	return l.env[name]
+}
+
+func lookupPathHost(cmd string, env map[string]string, writer io.Writer) (string, error) {
+	f, err := lookpath.LookPath2(cmd, &localEnv{env: env})
+	if err != nil {
+		err := "Cannot find: " + fmt.Sprint(cmd) + " in PATH"
+		if _, _err := writer.Write([]byte(err + "\n")); _err != nil {
+			return "", fmt.Errorf("%v: %w", err, _err)
+		}
+		return "", errors.New(err)
+	}
+	return f, nil
+}
+
+func setupPty(cmd *exec.Cmd, cmdline string) (*os.File, *os.File, error) {
+	ppty, tty, err := openPty()
+	if err != nil {
+		return nil, nil, err
+	}
+	if term.IsTerminal(int(tty.Fd())) {
+		_, err := term.MakeRaw(int(tty.Fd()))
+		if err != nil {
+			ppty.Close()
+			tty.Close()
+			return nil, nil, err
+		}
+	}
+	cmd.Stdin = tty
+	cmd.Stdout = tty
+	cmd.Stderr = tty
+	cmd.SysProcAttr = getSysProcAttr(cmdline, true)
+	return ppty, tty, nil
+}
+
+func writeKeepAlive(ppty io.Writer) {
+	c := 1
+	var err error
+	for c == 1 && err == nil {
+		c, err = ppty.Write([]byte{4})
+		<-time.After(time.Second)
+	}
+}
+
+func copyPtyOutput(writer io.Writer, ppty io.Reader, finishLog context.CancelFunc) {
+	defer func() {
+		finishLog()
+	}()
+	if _, err := io.Copy(writer, ppty); err != nil {
+		return
+	}
+}
+
+func (e *HostEnvironment) UpdateFromImageEnv(env *map[string]string) common.Executor {
+	return func(ctx context.Context) error {
+		return nil
+	}
+}
+
+func getEnvListFromMap(env map[string]string) []string {
+	envList := make([]string, 0)
+	for k, v := range env {
+		envList = append(envList, fmt.Sprintf("%s=%s", k, v))
+	}
+	return envList
+}
+
+func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline string, env map[string]string, user, workdir string) error {
+	envList := getEnvListFromMap(env)
+	var wd string
+	if workdir != "" {
+		if filepath.IsAbs(workdir) {
+			wd = workdir
+		} else {
+			wd = filepath.Join(e.Path, workdir)
+		}
+	} else {
+		wd = e.Path
+	}
+	f, err := lookupPathHost(command[0], env, e.StdOut)
+	if err != nil {
+		return err
+	}
+	cmd := exec.CommandContext(ctx, f)
+	cmd.Path = f
+	cmd.Args = command
+	cmd.Stdin = nil
+	cmd.Stdout = e.StdOut
+	cmd.Env = envList
+	cmd.Stderr = e.StdOut
+	cmd.Dir = wd
+	cmd.SysProcAttr = getSysProcAttr(cmdline, false)
+	var ppty *os.File
+	var tty *os.File
+	defer func() {
+		if ppty != nil {
+			ppty.Close()
+		}
+		if tty != nil {
+			tty.Close()
+		}
+	}()
+	if true /* allocate Terminal */ {
+		var err error
+		ppty, tty, err = setupPty(cmd, cmdline)
+		if err != nil {
+			common.Logger(ctx).Debugf("Failed to setup Pty %v\n", err.Error())
+		}
+	}
+	writer := &ptyWriter{Out: e.StdOut}
+	logctx, finishLog := context.WithCancel(context.Background())
+	if ppty != nil {
+		go copyPtyOutput(writer, ppty, finishLog)
+	} else {
+		finishLog()
+	}
+	if ppty != nil {
+		go writeKeepAlive(ppty)
+	}
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	if tty != nil {
+		writer.AutoStop = true
+		if _, err := tty.Write([]byte("\x04")); err != nil {
+			common.Logger(ctx).Debug("Failed to write EOT")
+		}
+	}
+	<-logctx.Done()
+
+	if ppty != nil {
+		ppty.Close()
+		ppty = nil
+	}
+	return err
+}
+
+func (e *HostEnvironment) Exec(command []string /*cmdline string, */, env map[string]string, user, workdir string) common.Executor {
+	return func(ctx context.Context) error {
+		if err := e.exec(ctx, command, "" /*cmdline*/, env, user, workdir); err != nil {
+			select {
+			case <-ctx.Done():
+				return fmt.Errorf("this step has been cancelled: %w", err)
+			default:
+				return err
+			}
+		}
+		return nil
+	}
+}
+
+func (e *HostEnvironment) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor {
+	localEnv := *env
+	return func(ctx context.Context) error {
+		envTar, err := e.GetContainerArchive(ctx, srcPath)
+		if err != nil {
+			return nil
+		}
+		defer envTar.Close()
+		reader := tar.NewReader(envTar)
+		_, err = reader.Next()
+		if err != nil && err != io.EOF {
+			return err
+		}
+		s := bufio.NewScanner(reader)
+		for s.Scan() {
+			line := s.Text()
+			singleLineEnv := strings.Index(line, "=")
+			multiLineEnv := strings.Index(line, "<<")
+			if singleLineEnv != -1 && (multiLineEnv == -1 || singleLineEnv < multiLineEnv) {
+				localEnv[line[:singleLineEnv]] = line[singleLineEnv+1:]
+			} else if multiLineEnv != -1 {
+				multiLineEnvContent := ""
+				multiLineEnvDelimiter := line[multiLineEnv+2:]
+				delimiterFound := false
+				for s.Scan() {
+					content := s.Text()
+					if content == multiLineEnvDelimiter {
+						delimiterFound = true
+						break
+					}
+					if multiLineEnvContent != "" {
+						multiLineEnvContent += "\n"
+					}
+					multiLineEnvContent += content
+				}
+				if !delimiterFound {
+					return fmt.Errorf("invalid format delimiter '%v' not found before end of file", multiLineEnvDelimiter)
+				}
+				localEnv[line[:multiLineEnv]] = multiLineEnvContent
+			} else {
+				return fmt.Errorf("invalid format '%v', expected a line with '=' or '<<'", line)
+			}
+		}
+		env = &localEnv
+		return nil
+	}
+}
+
+func (e *HostEnvironment) UpdateFromPath(env *map[string]string) common.Executor {
+	localEnv := *env
+	return func(ctx context.Context) error {
+		pathTar, err := e.GetContainerArchive(ctx, localEnv["GITHUB_PATH"])
+		if err != nil {
+			return err
+		}
+		defer pathTar.Close()
+
+		reader := tar.NewReader(pathTar)
+		_, err = reader.Next()
+		if err != nil && err != io.EOF {
+			return err
+		}
+		s := bufio.NewScanner(reader)
+		for s.Scan() {
+			line := s.Text()
+			pathSep := string(filepath.ListSeparator)
+			localEnv[e.GetPathVariableName()] = fmt.Sprintf("%s%s%s", line, pathSep, localEnv[e.GetPathVariableName()])
+		}
+
+		env = &localEnv
+		return nil
+	}
+}
+
+func (e *HostEnvironment) Remove() common.Executor {
+	return func(ctx context.Context) error {
+		if e.CleanUp != nil {
+			e.CleanUp()
+		}
+		return os.RemoveAll(e.Path)
+	}
+}
+
+func (e *HostEnvironment) ToContainerPath(path string) string {
+	if bp, err := filepath.Rel(e.Workdir, path); err != nil {
+		return filepath.Join(e.Path, bp)
+	} else if filepath.Clean(e.Workdir) == filepath.Clean(path) {
+		return e.Path
+	}
+	return path
+}
+
+func (e *HostEnvironment) GetActPath() string {
+	return e.ActPath
+}
+
+func (*HostEnvironment) GetPathVariableName() string {
+	if runtime.GOOS == "plan9" {
+		return "path"
+	} else if runtime.GOOS == "windows" {
+		return "Path" // Actually we need a case insensitive map
+	}
+	return "PATH"
+}
+
+func (e *HostEnvironment) DefaultPathVariable() string {
+	v, _ := os.LookupEnv(e.GetPathVariableName())
+	return v
+}
+
+func (*HostEnvironment) JoinPathVariable(paths ...string) string {
+	return strings.Join(paths, string(filepath.ListSeparator))
+}
+
+func (e *HostEnvironment) GetRunnerContext(ctx context.Context) map[string]interface{} {
+	return map[string]interface{}{
+		"os":         runtime.GOOS,
+		"arch":       runtime.GOARCH,
+		"temp":       e.TmpDir,
+		"tool_cache": e.ToolCache,
+	}
+}
+
+func (e *HostEnvironment) ReplaceLogWriter(stdout io.Writer, stderr io.Writer) (io.Writer, io.Writer) {
+	org := e.StdOut
+	e.StdOut = stdout
+	return org, org
+}

pkg/container/host_environment_test.go

@@ -0,0 +1,4 @@
+package container
+
+// Type assert HostEnvironment implements ExecutionsEnvironment
+var _ ExecutionsEnvironment = &HostEnvironment{}

pkg/container/linux_container_environment_extensions.go

@@ -0,0 +1,73 @@
+package container
+
+import (
+	"context"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type LinuxContainerEnvironmentExtensions struct {
+}
+
+// Resolves the equivalent host path inside the container
+// This is required for windows and WSL 2 to translate things like C:\Users\Myproject to /mnt/users/Myproject
+// For use in docker volumes and binds
+func (*LinuxContainerEnvironmentExtensions) ToContainerPath(path string) string {
+	if runtime.GOOS == "windows" && strings.Contains(path, "/") {
+		log.Error("You cannot specify linux style local paths (/mnt/etc) on Windows as it does not understand them.")
+		return ""
+	}
+
+	abspath, err := filepath.Abs(path)
+	if err != nil {
+		log.Error(err)
+		return ""
+	}
+
+	// Test if the path is a windows path
+	windowsPathRegex := regexp.MustCompile(`^([a-zA-Z]):\\(.+)$`)
+	windowsPathComponents := windowsPathRegex.FindStringSubmatch(abspath)
+
+	// Return as-is if no match
+	if windowsPathComponents == nil {
+		return abspath
+	}
+
+	// Convert to WSL2-compatible path if it is a windows path
+	// NOTE: Cannot use filepath because it will use the wrong path separators assuming we want the path to be windows
+	// based if running on Windows, and because we are feeding this to Docker, GoLang auto-path-translate doesn't work.
+	driveLetter := strings.ToLower(windowsPathComponents[1])
+	translatedPath := strings.ReplaceAll(windowsPathComponents[2], `\`, `/`)
+	// Should make something like /mnt/c/Users/person/My Folder/MyActProject
+	result := strings.Join([]string{"/mnt", driveLetter, translatedPath}, `/`)
+	return result
+}
+
+func (*LinuxContainerEnvironmentExtensions) GetActPath() string {
+	return "/var/run/act"
+}
+
+func (*LinuxContainerEnvironmentExtensions) GetPathVariableName() string {
+	return "PATH"
+}
+
+func (*LinuxContainerEnvironmentExtensions) DefaultPathVariable() string {
+	return "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+}
+
+func (*LinuxContainerEnvironmentExtensions) JoinPathVariable(paths ...string) string {
+	return strings.Join(paths, ":")
+}
+
+func (*LinuxContainerEnvironmentExtensions) GetRunnerContext(ctx context.Context) map[string]interface{} {
+	return map[string]interface{}{
+		"os":         "Linux",
+		"arch":       RunnerArch(ctx),
+		"temp":       "/tmp",
+		"tool_cache": "/opt/hostedtoolcache",
+	}
+}

pkg/container/linux_container_environment_extensions_test.go

@@ -0,0 +1,71 @@
+package container
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+	"strings"
+	"testing"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestContainerPath(t *testing.T) {
+	type containerPathJob struct {
+		destinationPath string
+		sourcePath      string
+		workDir         string
+	}
+
+	linuxcontainerext := &LinuxContainerEnvironmentExtensions{}
+
+	if runtime.GOOS == "windows" {
+		cwd, err := os.Getwd()
+		if err != nil {
+			log.Error(err)
+		}
+
+		rootDrive := os.Getenv("SystemDrive")
+		rootDriveLetter := strings.ReplaceAll(strings.ToLower(rootDrive), `:`, "")
+		for _, v := range []containerPathJob{
+			{"/mnt/c/Users/act/go/src/github.com/nektos/act", "C:\\Users\\act\\go\\src\\github.com\\nektos\\act\\", ""},
+			{"/mnt/f/work/dir", `F:\work\dir`, ""},
+			{"/mnt/c/windows/to/unix", "windows\\to\\unix", fmt.Sprintf("%s\\", rootDrive)},
+			{fmt.Sprintf("/mnt/%v/act", rootDriveLetter), "act", fmt.Sprintf("%s\\", rootDrive)},
+		} {
+			if v.workDir != "" {
+				if err := os.Chdir(v.workDir); err != nil {
+					log.Error(err)
+					t.Fail()
+				}
+			}
+
+			assert.Equal(t, v.destinationPath, linuxcontainerext.ToContainerPath(v.sourcePath))
+		}
+
+		if err := os.Chdir(cwd); err != nil {
+			log.Error(err)
+		}
+	} else {
+		cwd, err := os.Getwd()
+		if err != nil {
+			log.Error(err)
+		}
+		for _, v := range []containerPathJob{
+			{"/home/act/go/src/github.com/nektos/act", "/home/act/go/src/github.com/nektos/act", ""},
+			{"/home/act", `/home/act/`, ""},
+			{cwd, ".", ""},
+		} {
+			assert.Equal(t, v.destinationPath, linuxcontainerext.ToContainerPath(v.sourcePath))
+		}
+	}
+}
+
+type typeAssertMockContainer struct {
+	Container
+	LinuxContainerEnvironmentExtensions
+}
+
+// Type assert Container + LinuxContainerEnvironmentExtensions implements ExecutionsEnvironment
+var _ ExecutionsEnvironment = &typeAssertMockContainer{}

pkg/container/util.go

@@ -0,0 +1,26 @@
+//go:build (!windows && !plan9 && !openbsd) || (!windows && !plan9 && !mips64)
+
+package container
+
+import (
+	"os"
+	"syscall"
+
+	"github.com/creack/pty"
+)
+
+func getSysProcAttr(cmdLine string, tty bool) *syscall.SysProcAttr {
+	if tty {
+		return &syscall.SysProcAttr{
+			Setsid:  true,
+			Setctty: true,
+		}
+	}
+	return &syscall.SysProcAttr{
+		Setpgid: true,
+	}
+}
+
+func openPty() (*os.File, *os.File, error) {
+	return pty.Open()
+}

pkg/container/util_openbsd_mips64.go

@@ -0,0 +1,17 @@
+package container
+
+import (
+	"errors"
+	"os"
+	"syscall"
+)
+
+func getSysProcAttr(cmdLine string, tty bool) *syscall.SysProcAttr {
+	return &syscall.SysProcAttr{
+		Setpgid: true,
+	}
+}
+
+func openPty() (*os.File, *os.File, error) {
+	return nil, nil, errors.New("Unsupported")
+}

pkg/container/util_plan9.go

@@ -0,0 +1,17 @@
+package container
+
+import (
+	"errors"
+	"os"
+	"syscall"
+)
+
+func getSysProcAttr(cmdLine string, tty bool) *syscall.SysProcAttr {
+	return &syscall.SysProcAttr{
+		Rfork: syscall.RFNOTEG,
+	}
+}
+
+func openPty() (*os.File, *os.File, error) {
+	return nil, nil, errors.New("Unsupported")
+}

pkg/container/util_windows.go

@@ -0,0 +1,15 @@
+package container
+
+import (
+	"errors"
+	"os"
+	"syscall"
+)
+
+func getSysProcAttr(cmdLine string, tty bool) *syscall.SysProcAttr {
+	return &syscall.SysProcAttr{CmdLine: cmdLine, CreationFlags: syscall.CREATE_NEW_PROCESS_GROUP}
+}
+
+func openPty() (*os.File, *os.File, error) {
+	return nil, nil, errors.New("Unsupported")
+}