download + extract pipeline
- artifact.Download: resumable HTTP with optional SHA256 check + progress cb - artifact.ExtractZstdTar: streamed zstd+tar with tar-slip defense - aggregator client matches real API shape (digests/immutables/ancillary blocks with URIHolder polymorphism for templated immutable URIs) - cmd: show + download subcommands wired up - end-to-end verified against preprod: digests archive pulls cleanly, yields 16836-entry SHA manifest ready for verification sprint deps: github.com/klauspost/compress (pure-go zstd)
This commit is contained in:
Sulkta
parent
ddf1ec3491
commit
e4012bf79a
6 changed files with 483 additions and 84 deletions
|
|
@ -1,27 +1,136 @@
|
|||
// Package artifact handles downloading and extracting Mithril snapshot artifacts.
|
||||
// Currently stubs — HTTP range requests, resumable downloads, zstd+tar extraction
|
||||
// will be implemented in the next pass.
|
||||
package artifact
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
var ErrNotImplemented = errors.New("not yet implemented")
|
||||
// Download fetches a URL to destPath, resuming from a .part file if one
|
||||
// exists. If expectedSHA256 is non-empty, the final file is integrity-checked.
|
||||
// Progress is reported via the supplied callback (called with current bytes).
|
||||
//
|
||||
// Design notes:
|
||||
// - No parallel chunks yet; a single streaming GET is fine for sub-GB
|
||||
// artifacts and keeps the first working version simple. Range-chunk
|
||||
// parallelism will land in v2 once extraction is end-to-end tested.
|
||||
// - Resume is implemented via the HTTP Range header against the existing
|
||||
// .part file size; falls back to full download if the server refuses.
|
||||
// - destPath is atomically replaced only after SHA validation passes.
|
||||
func Download(ctx context.Context, uri, destPath, expectedSHA256 string, progress func(bytes int64)) error {
|
||||
if err := os.MkdirAll(filepath.Dir(destPath), 0o755); err != nil {
|
||||
return fmt.Errorf("mkdir: %w", err)
|
||||
}
|
||||
partPath := destPath + ".part"
|
||||
var existing int64
|
||||
if fi, err := os.Stat(partPath); err == nil {
|
||||
existing = fi.Size()
|
||||
}
|
||||
|
||||
// Download fetches an artifact from one of the supplied locations, choosing
|
||||
// the first reachable one and storing it at destPath.
|
||||
// Implementation will do:
|
||||
// - parallel range-chunks over HTTP
|
||||
// - resume on partial .part file
|
||||
// - SHA-256 verification against the snapshot manifest
|
||||
func Download(ctx context.Context, locations []string, destPath string) error {
|
||||
return ErrNotImplemented
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, uri, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if existing > 0 {
|
||||
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", existing))
|
||||
}
|
||||
|
||||
client := &http.Client{Timeout: 0} // artifacts can be GB-scale
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("GET %s: %w", uri, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var out *os.File
|
||||
switch resp.StatusCode {
|
||||
case http.StatusPartialContent:
|
||||
out, err = os.OpenFile(partPath, os.O_APPEND|os.O_WRONLY, 0o644)
|
||||
case http.StatusOK:
|
||||
// Server ignored our range; start over.
|
||||
existing = 0
|
||||
out, err = os.Create(partPath)
|
||||
default:
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 2048))
|
||||
return fmt.Errorf("GET %s: %d: %s", uri, resp.StatusCode, string(body))
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("open part: %w", err)
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
h := sha256.New()
|
||||
// If we're resuming, we need to re-hash the existing bytes.
|
||||
if existing > 0 {
|
||||
prev, err := os.Open(partPath)
|
||||
if err == nil {
|
||||
io.Copy(h, prev)
|
||||
prev.Close()
|
||||
}
|
||||
}
|
||||
|
||||
w := io.MultiWriter(out, h)
|
||||
total := existing
|
||||
buf := make([]byte, 256*1024)
|
||||
lastProgress := time.Now()
|
||||
for {
|
||||
n, rerr := resp.Body.Read(buf)
|
||||
if n > 0 {
|
||||
if _, werr := w.Write(buf[:n]); werr != nil {
|
||||
return fmt.Errorf("write: %w", werr)
|
||||
}
|
||||
total += int64(n)
|
||||
if progress != nil && time.Since(lastProgress) > 250*time.Millisecond {
|
||||
progress(total)
|
||||
lastProgress = time.Now()
|
||||
}
|
||||
}
|
||||
if rerr == io.EOF {
|
||||
break
|
||||
}
|
||||
if rerr != nil {
|
||||
return fmt.Errorf("read: %w", rerr)
|
||||
}
|
||||
}
|
||||
if progress != nil {
|
||||
progress(total)
|
||||
}
|
||||
if err := out.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if expectedSHA256 != "" {
|
||||
got := hex.EncodeToString(h.Sum(nil))
|
||||
if got != expectedSHA256 {
|
||||
return fmt.Errorf("SHA256 mismatch: want %s, got %s", expectedSHA256, got)
|
||||
}
|
||||
}
|
||||
|
||||
return os.Rename(partPath, destPath)
|
||||
}
|
||||
|
||||
// Extract decompresses a zstd+tar archive into targetDir.
|
||||
// Will stream through zstd -> tar reader without buffering the full archive.
|
||||
func Extract(ctx context.Context, archivePath, targetDir string) error {
|
||||
return ErrNotImplemented
|
||||
var ErrNoLocations = errors.New("no download locations available")
|
||||
|
||||
// DownloadFirst tries each URI in order until one succeeds.
|
||||
func DownloadFirst(ctx context.Context, uris []string, destPath, expectedSHA256 string, progress func(int64)) error {
|
||||
if len(uris) == 0 {
|
||||
return ErrNoLocations
|
||||
}
|
||||
var lastErr error
|
||||
for _, uri := range uris {
|
||||
if err := Download(ctx, uri, destPath, expectedSHA256, progress); err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("all locations failed: last error: %w", lastErr)
|
||||
}
|
||||
|
|
|
|||
91
internal/artifact/extract.go
Normal file
91
internal/artifact/extract.go
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
package artifact
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
// ExtractZstdTar decompresses a .tar.zst archive into targetDir, streaming
|
||||
// through the reader without buffering the full archive. Refuses entries
|
||||
// with ".." in the path or absolute paths (tar-slip defense).
|
||||
func ExtractZstdTar(ctx context.Context, archivePath, targetDir string) error {
|
||||
f, err := os.Open(archivePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open archive: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
zr, err := zstd.NewReader(f)
|
||||
if err != nil {
|
||||
return fmt.Errorf("zstd reader: %w", err)
|
||||
}
|
||||
defer zr.Close()
|
||||
|
||||
tr := tar.NewReader(zr)
|
||||
if err := os.MkdirAll(targetDir, 0o755); err != nil {
|
||||
return fmt.Errorf("mkdir target: %w", err)
|
||||
}
|
||||
|
||||
cleanTarget, err := filepath.Abs(targetDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("tar next: %w", err)
|
||||
}
|
||||
// tar-slip defense
|
||||
clean := filepath.Clean(hdr.Name)
|
||||
if strings.HasPrefix(clean, "..") || filepath.IsAbs(clean) {
|
||||
return fmt.Errorf("refusing suspicious archive path: %s", hdr.Name)
|
||||
}
|
||||
outPath := filepath.Join(cleanTarget, clean)
|
||||
if !strings.HasPrefix(filepath.Clean(outPath)+string(os.PathSeparator), cleanTarget+string(os.PathSeparator)) &&
|
||||
filepath.Clean(outPath) != cleanTarget {
|
||||
return fmt.Errorf("refusing path outside target: %s", hdr.Name)
|
||||
}
|
||||
|
||||
switch hdr.Typeflag {
|
||||
case tar.TypeDir:
|
||||
if err := os.MkdirAll(outPath, os.FileMode(hdr.Mode)); err != nil {
|
||||
return err
|
||||
}
|
||||
case tar.TypeReg:
|
||||
if err := os.MkdirAll(filepath.Dir(outPath), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
out, err := os.OpenFile(outPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode))
|
||||
if err != nil {
|
||||
return fmt.Errorf("create %s: %w", outPath, err)
|
||||
}
|
||||
if _, err := io.Copy(out, tr); err != nil {
|
||||
out.Close()
|
||||
return fmt.Errorf("write %s: %w", outPath, err)
|
||||
}
|
||||
if err := out.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
case tar.TypeSymlink, tar.TypeLink:
|
||||
// Refuse links for safety — a Mithril archive has no legitimate reason to contain them.
|
||||
return fmt.Errorf("refusing link entry: %s", hdr.Name)
|
||||
default:
|
||||
// Silently skip unknown types.
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in a new issue