codingworkshop/act
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: cli option to enable the new action cache (#1954)

Browse Source 
* Enable the new action cache

* fix

* fix: CopyTarStream (Docker)

* suppress panic in test

* add a cli option for opt in

* fixups

* add package

* fix

* rc.Config nil in test???

* add feature flag

* patch

* Fix respect --action-cache-path

Co-authored-by: Björn Brauer <zaubernerd@zaubernerd.de>

* add remote reusable workflow to ActionCache

* fixup

---------

Co-authored-by: Björn Brauer <zaubernerd@zaubernerd.de>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
ChristopherHX
2024-01-20 00:49:35 +01:00
committed by GitHub
parent cd40f3fe9b
commit f7a846d2f5
10 changed files with 211 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
pkg/runner/action.go
View File

@@ -44,7 +44,7 @@ func readActionImpl(ctx context.Context, step *model.Step, actionDir string, act
reader, closer, err := readFile("action.yml")
if os.IsNotExist(err) {
reader, closer, err = readFile("action.yaml")
if err != nil {
if os.IsNotExist(err) {
if _, closer, err2 := readFile("Dockerfile"); err2 == nil {
closer.Close()
action := &model.Action{
@@ -91,6 +91,8 @@ func readActionImpl(ctx context.Context, step *model.Step, actionDir string, act
}
}
return nil, err
} else if err != nil {
return nil, err
}
} else if err != nil {
return nil, err
@@ -110,6 +112,17 @@ func maybeCopyToActionDir(ctx context.Context, step actionStep, actionDir string
if stepModel.Type() != model.StepTypeUsesActionRemote {
return nil
}
if rc.Config != nil && rc.Config.ActionCache != nil {
raction := step.(*stepActionRemote)
ta, err := rc.Config.ActionCache.GetTarArchive(ctx, raction.cacheDir, raction.resolvedSha, "")
if err != nil {
return err
}
defer ta.Close()
return rc.JobContainer.CopyTarStream(ctx, containerActionDir, ta)
}
if err := removeGitIgnore(ctx, actionDir); err != nil {
return err
}
@@ -265,6 +278,13 @@ func execAsDocker(ctx context.Context, step actionStep, actionName string, based
return err
}
defer buildContext.Close()
} else if rc.Config.ActionCache != nil {
rstep := step.(*stepActionRemote)
buildContext, err = rc.Config.ActionCache.GetTarArchive(ctx, rstep.cacheDir, rstep.resolvedSha, contextDir)
if err != nil {
return err
}
defer buildContext.Close()
}
prepImage = container.NewDockerBuildExecutor(container.NewDockerBuildExecutorInput{
ContextDir: contextDir,

40
pkg/runner/reusable_workflow.go
View File

@@ -1,6 +1,7 @@
package runner
import (
"archive/tar"
"context"
"errors"
"fmt"
@@ -33,12 +34,51 @@ func newRemoteReusableWorkflowExecutor(rc *RunContext) common.Executor {
filename := fmt.Sprintf("%s/%s@%s", remoteReusableWorkflow.Org, remoteReusableWorkflow.Repo, remoteReusableWorkflow.Ref)
workflowDir := fmt.Sprintf("%s/%s", rc.ActionCacheDir(), safeFilename(filename))
if rc.Config.ActionCache != nil {
return newActionCacheReusableWorkflowExecutor(rc, filename, remoteReusableWorkflow)
}
return common.NewPipelineExecutor(
newMutexExecutor(cloneIfRequired(rc, *remoteReusableWorkflow, workflowDir)),
newReusableWorkflowExecutor(rc, workflowDir, fmt.Sprintf("./.github/workflows/%s", remoteReusableWorkflow.Filename)),
)
}
func newActionCacheReusableWorkflowExecutor(rc *RunContext, filename string, remoteReusableWorkflow *remoteReusableWorkflow) common.Executor {
return func(ctx context.Context) error {
ghctx := rc.getGithubContext(ctx)
remoteReusableWorkflow.URL = ghctx.ServerURL
sha, err := rc.Config.ActionCache.Fetch(ctx, filename, remoteReusableWorkflow.CloneURL(), remoteReusableWorkflow.Ref, ghctx.Token)
if err != nil {
return err
}
archive, err := rc.Config.ActionCache.GetTarArchive(ctx, filename, sha, fmt.Sprintf(".github/workflows/%s", remoteReusableWorkflow.Filename))
if err != nil {
return err
}
defer archive.Close()
treader := tar.NewReader(archive)
if _, err = treader.Next(); err != nil {
return err
}
planner, err := model.NewSingleWorkflowPlanner(remoteReusableWorkflow.Filename, treader)
if err != nil {
return err
}
plan, err := planner.PlanEvent("workflow_call")
if err != nil {
return err
}
runner, err := NewReusableWorkflowRunner(rc)
if err != nil {
return err
}
return runner.NewPlanExecutor(plan)(ctx)
}
}
var (
executorLock sync.Mutex
)

1
pkg/runner/runner.go
View File

@@ -59,6 +59,7 @@ type Config struct {
ReplaceGheActionTokenWithGithubCom string // Token of private action repo on GitHub.
Matrix map[string]map[string]bool // Matrix config to run
ContainerNetworkMode docker_container.NetworkMode // the network mode of job containers (the value of --network)
ActionCache ActionCache // Use a custom ActionCache Implementation
}
type caller struct {

13
pkg/runner/step.go
View File

@@ -34,6 +34,9 @@ const (
stepStagePost
)
// Controls how many symlinks are resolved for local and remote Actions
const maxSymlinkDepth = 10
func (s stepStage) String() string {
switch s {
case stepStagePre:
@@ -307,3 +310,13 @@ func mergeIntoMapCaseInsensitive(target map[string]string, maps ...map[string]st
}
}
}
func symlinkJoin(filename, sym, parent string) (string, error) {
dir := path.Dir(filename)
dest := path.Join(dir, sym)
prefix := path.Clean(parent) + "/"
if strings.HasPrefix(dest, prefix) || prefix == "./" {
return dest, nil
}
return "", fmt.Errorf("symlink tries to access file '%s' outside of '%s'", strings.ReplaceAll(dest, "'", "''"), strings.ReplaceAll(parent, "'", "''"))
}

35
pkg/runner/step_action_local.go
View File

@@ -3,7 +3,10 @@ package runner
import (
"archive/tar"
"context"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path"
"path/filepath"
@@ -42,15 +45,31 @@ func (sal *stepActionLocal) main() common.Executor {
localReader := func(ctx context.Context) actionYamlReader {
_, cpath := getContainerActionPaths(sal.Step, path.Join(actionDir, ""), sal.RunContext)
return func(filename string) (io.Reader, io.Closer, error) {
tars, err := sal.RunContext.JobContainer.GetContainerArchive(ctx, path.Join(cpath, filename))
if err != nil {
return nil, nil, os.ErrNotExist
spath := path.Join(cpath, filename)
for i := 0; i < maxSymlinkDepth; i++ {
tars, err := sal.RunContext.JobContainer.GetContainerArchive(ctx, spath)
if errors.Is(err, fs.ErrNotExist) {
return nil, nil, err
} else if err != nil {
return nil, nil, fs.ErrNotExist
}
treader := tar.NewReader(tars)
header, err := treader.Next()
if errors.Is(err, io.EOF) {
return nil, nil, os.ErrNotExist
} else if err != nil {
return nil, nil, err
}
if header.FileInfo().Mode()&os.ModeSymlink == os.ModeSymlink {
spath, err = symlinkJoin(spath, header.Linkname, cpath)
if err != nil {
return nil, nil, err
}
} else {
return treader, tars, nil
}
}
treader := tar.NewReader(tars)
if _, err := treader.Next(); err != nil {
return nil, nil, os.ErrNotExist
}
return treader, tars, nil
return nil, nil, fmt.Errorf("max depth %d of symlinks exceeded while reading %s", maxSymlinkDepth, spath)
}
}

43
pkg/runner/step_action_remote.go
View File

@@ -1,6 +1,7 @@
package runner
import (
"archive/tar"
"context"
"errors"
"fmt"
@@ -28,6 +29,8 @@ type stepActionRemote struct {
action *model.Action
env map[string]string
remoteAction *remoteAction
cacheDir string
resolvedSha string
}
var (
@@ -60,6 +63,46 @@ func (sar *stepActionRemote) prepareActionExecutor() common.Executor {
github.Token = sar.RunContext.Config.ReplaceGheActionTokenWithGithubCom
}
}
if sar.RunContext.Config.ActionCache != nil {
cache := sar.RunContext.Config.ActionCache
var err error
sar.cacheDir = fmt.Sprintf("%s/%s", sar.remoteAction.Org, sar.remoteAction.Repo)
sar.resolvedSha, err = cache.Fetch(ctx, sar.cacheDir, sar.remoteAction.URL+"/"+sar.cacheDir, sar.remoteAction.Ref, github.Token)
if err != nil {
return err
}
remoteReader := func(ctx context.Context) actionYamlReader {
return func(filename string) (io.Reader, io.Closer, error) {
spath := filename
for i := 0; i < maxSymlinkDepth; i++ {
tars, err := cache.GetTarArchive(ctx, sar.cacheDir, sar.resolvedSha, spath)
if err != nil {
return nil, nil, os.ErrNotExist
}
treader := tar.NewReader(tars)
header, err := treader.Next()
if err != nil {
return nil, nil, os.ErrNotExist
}
if header.FileInfo().Mode()&os.ModeSymlink == os.ModeSymlink {
spath, err = symlinkJoin(spath, header.Linkname, ".")
if err != nil {
return nil, nil, err
}
} else {
return treader, tars, nil
}
}
return nil, nil, fmt.Errorf("max depth %d of symlinks exceeded while reading %s", maxSymlinkDepth, spath)
}
}
actionModel, err := sar.readAction(ctx, sar.Step, sar.resolvedSha, sar.remoteAction.Path, remoteReader(ctx), os.WriteFile)
sar.action = actionModel
return err
}
actionDir := fmt.Sprintf("%s/%s", sar.RunContext.ActionCacheDir(), safeFilename(sar.Step.Uses))
gitClone := stepActionRemoteNewCloneExecutor(git.NewGitCloneExecutorInput{