chore: establish local platform baseline

2026-08-26 06:41:54 +08:00
commit 29bdf4900f
575 changed files with 95644 additions and 0 deletions
@@ -0,0 +1,177 @@
+package objectstore
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"net/url"
+	"time"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+var (
+	ErrObjectSizeMismatch   = errors.New("object size mismatch")
+	ErrObjectDigestMismatch = errors.New("object digest mismatch")
+)
+
+type Store struct {
+	internal *minio.Client
+	public   *minio.Client
+	bucket   string
+}
+
+func New(endpoint, publicEndpoint, accessKey, secretKey, bucket, region string, secure bool) (*Store, error) {
+	if endpoint == "" || publicEndpoint == "" || accessKey == "" || secretKey == "" || bucket == "" || region == "" {
+		return nil, errors.New("object store endpoints, credentials, bucket and region are required")
+	}
+	options := &minio.Options{Creds: credentials.NewStaticV4(accessKey, secretKey, ""), Secure: secure, Region: region}
+	internalClient, err := minio.New(endpoint, options)
+	if err != nil {
+		return nil, fmt.Errorf("create internal object store client: %w", err)
+	}
+	publicClient, err := minio.New(publicEndpoint, options)
+	if err != nil {
+		return nil, fmt.Errorf("create public object store signer: %w", err)
+	}
+	return &Store{internal: internalClient, public: publicClient, bucket: bucket}, nil
+}
+
+func (s *Store) EnsureBucket(ctx context.Context) error {
+	exists, err := s.internal.BucketExists(ctx, s.bucket)
+	if err != nil {
+		return fmt.Errorf("check export bucket: %w", err)
+	}
+	if exists {
+		return nil
+	}
+	if err := s.internal.MakeBucket(ctx, s.bucket, minio.MakeBucketOptions{}); err != nil {
+		exists, checkErr := s.internal.BucketExists(ctx, s.bucket)
+		if checkErr == nil && exists {
+			return nil
+		}
+		return fmt.Errorf("create export bucket: %w", err)
+	}
+	return nil
+}
+
+func (s *Store) Ready(ctx context.Context) error {
+	exists, err := s.internal.BucketExists(ctx, s.bucket)
+	if err != nil {
+		return err
+	}
+	if !exists {
+		return errors.New("export bucket does not exist")
+	}
+	return nil
+}
+
+func (s *Store) PutJSON(ctx context.Context, objectRef string, content []byte, digest string) (int64, error) {
+	result, err := s.internal.PutObject(ctx, s.bucket, objectRef, bytes.NewReader(content), int64(len(content)), minio.PutObjectOptions{
+		ContentType:  "application/json; charset=utf-8",
+		UserMetadata: map[string]string{"content-sha256": digest, "data-classification": "private"},
+	})
+	if err != nil {
+		return 0, fmt.Errorf("put private export object: %w", err)
+	}
+	return result.Size, nil
+}
+
+func (s *Store) PresignDownload(ctx context.Context, objectRef string, ttl time.Duration, filename string) (string, error) {
+	if ttl <= 0 {
+		return "", errors.New("download TTL must be positive")
+	}
+	parameters := make(url.Values)
+	parameters.Set("response-content-disposition", fmt.Sprintf(`attachment; filename="%s"`, filename))
+	parameters.Set("response-content-type", "application/json; charset=utf-8")
+	signed, err := s.public.PresignedGetObject(ctx, s.bucket, objectRef, ttl, parameters)
+	if err != nil {
+		return "", fmt.Errorf("sign private export download: %w", err)
+	}
+	return signed.String(), nil
+}
+
+func (s *Store) PresignUpload(ctx context.Context, objectRef string, ttl time.Duration) (string, error) {
+	if ttl <= 0 {
+		return "", errors.New("upload TTL must be positive")
+	}
+	signed, err := s.public.PresignedPutObject(ctx, s.bucket, objectRef, ttl)
+	if err != nil {
+		return "", fmt.Errorf("sign private firmware upload: %w", err)
+	}
+	return signed.String(), nil
+}
+
+func (s *Store) VerifyDigest(ctx context.Context, objectRef, expectedDigest string, expectedSize int64) error {
+	object, err := s.internal.GetObject(ctx, s.bucket, objectRef, minio.GetObjectOptions{})
+	if err != nil {
+		return fmt.Errorf("open private firmware object: %w", err)
+	}
+	defer object.Close()
+	hash := sha256.New()
+	written, err := io.Copy(hash, object)
+	if err != nil {
+		return fmt.Errorf("read private firmware object: %w", err)
+	}
+	if written != expectedSize {
+		return fmt.Errorf("%w: got %d expected %d", ErrObjectSizeMismatch, written, expectedSize)
+	}
+	actual := hex.EncodeToString(hash.Sum(nil))
+	if actual != expectedDigest {
+		return ErrObjectDigestMismatch
+	}
+	return nil
+}
+
+func (s *Store) ReadPrefix(ctx context.Context, objectRef string, limit int64) ([]byte, error) {
+	if limit < 1 || limit > 1024*1024 {
+		return nil, errors.New("object prefix limit must be between 1 byte and 1 MiB")
+	}
+	object, err := s.internal.GetObject(ctx, s.bucket, objectRef, minio.GetObjectOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("open private media object: %w", err)
+	}
+	defer object.Close()
+	content, err := io.ReadAll(io.LimitReader(object, limit))
+	if err != nil {
+		return nil, fmt.Errorf("read private media object prefix: %w", err)
+	}
+	return content, nil
+}
+
+func (s *Store) PresignMediaDownload(ctx context.Context, objectRef string, ttl time.Duration, filename, contentType string, inline bool) (string, error) {
+	if ttl <= 0 {
+		return "", errors.New("download TTL must be positive")
+	}
+	disposition := "attachment"
+	if inline {
+		disposition = "inline"
+	}
+	parameters := make(url.Values)
+	parameters.Set("response-content-disposition", fmt.Sprintf(`%s; filename="%s"`, disposition, filename))
+	parameters.Set("response-content-type", contentType)
+	signed, err := s.public.PresignedGetObject(ctx, s.bucket, objectRef, ttl, parameters)
+	if err != nil {
+		return "", fmt.Errorf("sign private media download: %w", err)
+	}
+	return signed.String(), nil
+}
+
+func (s *Store) Remove(ctx context.Context, objectRef string) error {
+	if objectRef == "" {
+		return nil
+	}
+	if err := s.internal.RemoveObject(ctx, s.bucket, objectRef, minio.RemoveObjectOptions{}); err != nil {
+		return fmt.Errorf("remove private export object: %w", err)
+	}
+	return nil
+}
+
+func (s *Store) Stat(ctx context.Context, objectRef string) (minio.ObjectInfo, error) {
+	return s.internal.StatObject(ctx, s.bucket, objectRef, minio.StatObjectOptions{})
+}
@@ -0,0 +1,108 @@
+package objectstore
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+)
+
+func TestStoreValidationAndOfflineSigning(t *testing.T) {
+	if _, err := New("", "localhost:59000", "key", "secret", "bucket", "us-east-1", false); err == nil {
+		t.Fatal("missing internal endpoint was accepted")
+	}
+	store, err := New("localhost:59000", "localhost:59000", "local_minio", "local_minio_password", "private-exports", "us-east-1", false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx := context.Background()
+	if _, err := store.PresignDownload(ctx, "synthetic/object.json", 0, "export.json"); err == nil {
+		t.Fatal("non-positive download TTL was accepted")
+	}
+	download, err := store.PresignDownload(ctx, "synthetic/object.json", time.Minute, "export.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	parsed, err := url.Parse(download)
+	if err != nil || parsed.Query().Get("X-Amz-Signature") == "" || parsed.Query().Get("response-content-disposition") != `attachment; filename="export.json"` {
+		t.Fatalf("download URL=%q parsed=%+v err=%v", download, parsed, err)
+	}
+	if _, err := store.PresignUpload(ctx, "synthetic/firmware.bin", 0); err == nil {
+		t.Fatal("non-positive upload TTL was accepted")
+	}
+	upload, err := store.PresignUpload(ctx, "synthetic/firmware.bin", time.Minute)
+	if err != nil || !strings.Contains(upload, "X-Amz-Signature=") {
+		t.Fatalf("upload URL=%q err=%v", upload, err)
+	}
+	mediaURL, err := store.PresignMediaDownload(ctx, "synthetic/audio.wav", time.Minute, "audio.wav", "audio/wav", true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	parsed, _ = url.Parse(mediaURL)
+	if parsed.Query().Get("response-content-disposition") != `inline; filename="audio.wav"` || parsed.Query().Get("response-content-type") != "audio/wav" {
+		t.Fatalf("media URL=%q", mediaURL)
+	}
+	if _, err := store.PresignMediaDownload(ctx, "synthetic/audio.wav", -time.Second, "audio.wav", "audio/wav", false); err == nil {
+		t.Fatal("negative media TTL was accepted")
+	}
+	if _, err := store.ReadPrefix(ctx, "synthetic/audio.wav", 0); err == nil {
+		t.Fatal("invalid prefix limit was accepted")
+	}
+	if err := store.Remove(ctx, ""); err != nil {
+		t.Fatalf("empty remove: %v", err)
+	}
+}
+
+func TestStoreMinIOObjectLifecycle(t *testing.T) {
+	endpoint := os.Getenv("TEST_OBJECT_STORE_ENDPOINT")
+	if endpoint == "" {
+		t.Skip("TEST_OBJECT_STORE_ENDPOINT is not set")
+	}
+	store, err := New(endpoint, endpoint, "local_minio", "local_minio_password", "private-exports", "us-east-1", false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx := context.Background()
+	if err := store.EnsureBucket(ctx); err != nil {
+		t.Fatal(err)
+	}
+	if err := store.Ready(ctx); err != nil {
+		t.Fatal(err)
+	}
+	objectRef := "integration/objectstore/" + uuid.NewString() + ".json"
+	defer store.Remove(ctx, objectRef)
+	content := []byte(`{"synthetic":true,"message":"object lifecycle"}`)
+	digestBytes := sha256.Sum256(content)
+	digest := hex.EncodeToString(digestBytes[:])
+	size, err := store.PutJSON(ctx, objectRef, content, digest)
+	if err != nil || size != int64(len(content)) {
+		t.Fatalf("put size=%d err=%v", size, err)
+	}
+	info, err := store.Stat(ctx, objectRef)
+	if err != nil || info.Size != int64(len(content)) {
+		t.Fatalf("stat=%+v err=%v", info, err)
+	}
+	prefix, err := store.ReadPrefix(ctx, objectRef, 12)
+	if err != nil || string(prefix) != string(content[:12]) {
+		t.Fatalf("prefix=%q err=%v", prefix, err)
+	}
+	if err := store.VerifyDigest(ctx, objectRef, digest, int64(len(content)+1)); !errors.Is(err, ErrObjectSizeMismatch) {
+		t.Fatalf("size mismatch error=%v", err)
+	}
+	if err := store.VerifyDigest(ctx, objectRef, strings.Repeat("0", 64), int64(len(content))); !errors.Is(err, ErrObjectDigestMismatch) {
+		t.Fatalf("digest mismatch error=%v", err)
+	}
+	if err := store.VerifyDigest(ctx, objectRef, digest, int64(len(content))); err != nil {
+		t.Fatalf("verify object: %v", err)
+	}
+	if err := store.Remove(ctx, objectRef); err != nil {
+		t.Fatal(err)
+	}
+}