download + extract pipeline
- artifact.Download: resumable HTTP with optional SHA256 check + progress cb - artifact.ExtractZstdTar: streamed zstd+tar with tar-slip defense - aggregator client matches real API shape (digests/immutables/ancillary blocks with URIHolder polymorphism for templated immutable URIs) - cmd: show + download subcommands wired up - end-to-end verified against preprod: digests archive pulls cleanly, yields 16836-entry SHA manifest ready for verification sprint deps: github.com/klauspost/compress (pure-go zstd)
This commit is contained in:
Sulkta
parent
ddf1ec3491
commit
e4012bf79a
6 changed files with 483 additions and 84 deletions
|
|
@ -1,7 +1,7 @@
|
|||
// Package aggregator is a thin HTTP client for the Mithril aggregator REST API.
|
||||
//
|
||||
// Only the handful of endpoints needed for client-side snapshot workflows are
|
||||
// exposed. Authentication is not required for the read paths used here.
|
||||
// Only the endpoints needed for client-side snapshot workflows are exposed.
|
||||
// Authentication is not required for the read paths used here.
|
||||
package aggregator
|
||||
|
||||
import (
|
||||
|
|
@ -23,57 +23,98 @@ type Client struct {
|
|||
func New(baseURL string) *Client {
|
||||
return &Client{
|
||||
baseURL: strings.TrimRight(baseURL, "/"),
|
||||
http: &http.Client{Timeout: 60 * time.Second},
|
||||
http: &http.Client{Timeout: 120 * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
// CardanoDBSnapshot is the server-reported shape for /artifact/cardano-database/{hash}.
|
||||
// Field set is trimmed to what the client actually consumes — full schema documented
|
||||
// at https://mithril.network/doc/aggregator-api/.
|
||||
// CardanoDBSnapshot is the server shape for /artifact/cardano-database and its
|
||||
// /{hash} detail endpoint. The list response omits {digests,immutables,ancillary};
|
||||
// only the detail endpoint populates them.
|
||||
type CardanoDBSnapshot struct {
|
||||
Hash string `json:"hash"`
|
||||
MerkleRoot string `json:"merkle_root"`
|
||||
Network string `json:"network"`
|
||||
Beacon Beacon `json:"beacon"`
|
||||
CertificateHash string `json:"certificate_hash"`
|
||||
TotalDBSizeUncompressed uint64 `json:"total_db_size_uncompressed"`
|
||||
Digests LocationList `json:"digests"`
|
||||
ImmutablesAncillary LocationList `json:"immutables"`
|
||||
ImmutablesIncremental *IncrementalImmutables `json:"immutables_incremental,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Hash string `json:"hash"`
|
||||
MerkleRoot string `json:"merkle_root"`
|
||||
Network string `json:"network"`
|
||||
Beacon Beacon `json:"beacon"`
|
||||
CertificateHash string `json:"certificate_hash"`
|
||||
TotalDBSizeUncompressed uint64 `json:"total_db_size_uncompressed"`
|
||||
CardanoNodeVersion string `json:"cardano_node_version"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Digests DigestsBlock `json:"digests"`
|
||||
Immutables ImmutsBlock `json:"immutables"`
|
||||
Ancillary AncillaryBlock `json:"ancillary"`
|
||||
}
|
||||
|
||||
type Beacon struct {
|
||||
Epoch uint64 `json:"epoch"`
|
||||
Epoch uint64 `json:"epoch"`
|
||||
ImmutableFileNumber uint64 `json:"immutable_file_number"`
|
||||
}
|
||||
|
||||
type LocationList struct {
|
||||
Size uint64 `json:"size"`
|
||||
Locations []LocationAlt `json:"locations"`
|
||||
type DigestsBlock struct {
|
||||
SizeUncompressed uint64 `json:"size_uncompressed"`
|
||||
Locations []Location `json:"locations"`
|
||||
}
|
||||
|
||||
// LocationAlt is a best-of alternative; Mithril returns a typed-discriminated object.
|
||||
type LocationAlt struct {
|
||||
Type string `json:"type"` // e.g. "cloud_storage", "ipfs"
|
||||
URI string `json:"uri"`
|
||||
type ImmutsBlock struct {
|
||||
AverageSizeUncompressed uint64 `json:"average_size_uncompressed"`
|
||||
Locations []Location `json:"locations"`
|
||||
}
|
||||
|
||||
type IncrementalImmutables struct {
|
||||
AverageSize uint64 `json:"average_size"`
|
||||
Locations []LocationAlt `json:"locations"`
|
||||
type AncillaryBlock struct {
|
||||
SizeUncompressed uint64 `json:"size_uncompressed"`
|
||||
Locations []Location `json:"locations"`
|
||||
}
|
||||
|
||||
// Location is a polymorphic URI holder. The Mithril API ships URI as either
|
||||
// a plain string (for single artifacts) or as {"Template": "..."} for
|
||||
// templated per-file URIs (immutables only).
|
||||
type Location struct {
|
||||
Type string `json:"type"`
|
||||
URI URIHolder `json:"uri"`
|
||||
CompressionAlgorithm string `json:"compression_algorithm,omitempty"`
|
||||
}
|
||||
|
||||
// URIHolder absorbs both string and templated-object URI shapes.
|
||||
type URIHolder struct {
|
||||
Plain string
|
||||
Template string
|
||||
}
|
||||
|
||||
func (h *URIHolder) UnmarshalJSON(b []byte) error {
|
||||
// Try plain string first
|
||||
var s string
|
||||
if err := json.Unmarshal(b, &s); err == nil {
|
||||
h.Plain = s
|
||||
return nil
|
||||
}
|
||||
// Fall back to {"Template": "..."}
|
||||
var t struct {
|
||||
Template string `json:"Template"`
|
||||
}
|
||||
if err := json.Unmarshal(b, &t); err == nil {
|
||||
h.Template = t.Template
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("unrecognized URI shape: %s", string(b))
|
||||
}
|
||||
|
||||
// String returns whichever URI form is populated.
|
||||
func (h URIHolder) String() string {
|
||||
if h.Template != "" {
|
||||
return h.Template
|
||||
}
|
||||
return h.Plain
|
||||
}
|
||||
|
||||
// Certificate is the server-reported shape for /certificate/{hash}.
|
||||
// Kept minimal; STM verification reads what it needs from the raw JSON later.
|
||||
// Kept wide — STM verification consumes raw bytes separately from the decoded view.
|
||||
type Certificate struct {
|
||||
Hash string `json:"hash"`
|
||||
PreviousHash string `json:"previous_hash"`
|
||||
Epoch uint64 `json:"epoch"`
|
||||
SignedMessage string `json:"signed_message"`
|
||||
ProtocolMessage json.RawMessage `json:"protocol_message"`
|
||||
Multisignature json.RawMessage `json:"multi_signature"`
|
||||
GenesisSignature string `json:"genesis_signature,omitempty"`
|
||||
Hash string `json:"hash"`
|
||||
PreviousHash string `json:"previous_hash"`
|
||||
Epoch uint64 `json:"epoch"`
|
||||
SignedMessage string `json:"signed_message"`
|
||||
ProtocolMessage json.RawMessage `json:"protocol_message"`
|
||||
Multisignature json.RawMessage `json:"multi_signature"`
|
||||
GenesisSignature string `json:"genesis_signature,omitempty"`
|
||||
}
|
||||
|
||||
func (c *Client) get(ctx context.Context, path string, out any) error {
|
||||
|
|
@ -84,12 +125,12 @@ func (c *Client) get(ctx context.Context, path string, out any) error {
|
|||
req.Header.Set("Accept", "application/json")
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("aggregator GET %s: %w", path, err)
|
||||
return fmt.Errorf("GET %s: %w", path, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 400 {
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||||
return fmt.Errorf("aggregator GET %s: status %d: %s", path, resp.StatusCode, string(body))
|
||||
return fmt.Errorf("GET %s: %d: %s", path, resp.StatusCode, string(body))
|
||||
}
|
||||
if out == nil {
|
||||
return nil
|
||||
|
|
@ -97,16 +138,11 @@ func (c *Client) get(ctx context.Context, path string, out any) error {
|
|||
return json.NewDecoder(resp.Body).Decode(out)
|
||||
}
|
||||
|
||||
// ListCardanoDBSnapshots returns the sorted-newest-first list of cardano-database snapshots.
|
||||
func (c *Client) ListCardanoDBSnapshots(ctx context.Context) ([]CardanoDBSnapshot, error) {
|
||||
var out []CardanoDBSnapshot
|
||||
if err := c.get(ctx, "/artifact/cardano-database", &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
return out, c.get(ctx, "/artifact/cardano-database", &out)
|
||||
}
|
||||
|
||||
// GetCardanoDBSnapshot fetches details for a single snapshot by hash (or "latest").
|
||||
func (c *Client) GetCardanoDBSnapshot(ctx context.Context, hash string) (*CardanoDBSnapshot, error) {
|
||||
var out CardanoDBSnapshot
|
||||
if err := c.get(ctx, "/artifact/cardano-database/"+url.PathEscape(hash), &out); err != nil {
|
||||
|
|
@ -115,7 +151,6 @@ func (c *Client) GetCardanoDBSnapshot(ctx context.Context, hash string) (*Cardan
|
|||
return &out, nil
|
||||
}
|
||||
|
||||
// GetCertificate fetches a certificate by hash for signature verification.
|
||||
func (c *Client) GetCertificate(ctx context.Context, hash string) (*Certificate, error) {
|
||||
var out Certificate
|
||||
if err := c.get(ctx, "/certificate/"+url.PathEscape(hash), &out); err != nil {
|
||||
|
|
|
|||
Reference in a new issue