From 4ae20f98964e43ac6e6be2ea226e1e7c67698891 Mon Sep 17 00:00:00 2001 From: Xe Iaso Date: Thu, 28 May 2026 12:32:49 -0400 Subject: [PATCH] feat(s3fs): add optional Unix-metadata storage as S3 user metadata Store POSIX file attributes (uid, gid, mode, mtime) as x-amz-meta-* headers on S3 objects so POSIX filesystems layered on Tigris can round-trip ownership and permissions instead of always reporting mode 0666 with no owner. The feature is opt-in and off by default: NewS3FS takes functional options, and without WithUnixMetadata the filesystem behaves exactly as before. Callers opt in by passing WithUnixMetadata(uid, gid, umask) and resolve any name strings to numeric IDs themselves. New unixmeta package implements Encode/Decode and PosixMode/GoFileMode conversion; the wire format is documented in docs/reference. Assisted-by: Claude Opus 4.7 via Claude Code Signed-off-by: Xe Iaso --- docs/plans/s3fs-unix-metadata.md | 145 +++++++++ docs/reference/how-tigris-fs-unix-metadata.md | 281 ++++++++++++++++++ internal/s3fs/basic.go | 11 +- internal/s3fs/chroot.go | 1 + internal/s3fs/file.go | 59 ++-- internal/s3fs/fileinfo.go | 52 +++- internal/s3fs/filesystem.go | 34 ++- internal/s3fs/unixmeta/unixmeta.go | 207 +++++++++++++ internal/s3fs/unixmeta/unixmeta_test.go | 132 ++++++++ 9 files changed, 887 insertions(+), 35 deletions(-) create mode 100644 docs/plans/s3fs-unix-metadata.md create mode 100644 docs/reference/how-tigris-fs-unix-metadata.md create mode 100644 internal/s3fs/unixmeta/unixmeta.go create mode 100644 internal/s3fs/unixmeta/unixmeta_test.go diff --git a/docs/plans/s3fs-unix-metadata.md b/docs/plans/s3fs-unix-metadata.md new file mode 100644 index 0000000..29f5fa6 --- /dev/null +++ b/docs/plans/s3fs-unix-metadata.md @@ -0,0 +1,145 @@ +# Plan: Optional Unix-permission metadata in s3fs + +## Context + +`internal/s3fs/` is a go-billy filesystem mapped onto Tigris (S3) storage, +consumed today by `cmd/objgitd` to back git repositories. It never reads or +writes POSIX attributes: every file reports mode `0666`, directories `ModeDir`, +uid/gid are absent, and `PutObject` carries no user metadata +(`internal/s3fs/fileinfo.go`, `internal/s3fs/file.go`). + +`docs/reference/how-tigris-fs-unix-metadata.md` defines a convention for storing +Unix attributes as `x-amz-meta-*` headers (uid, gid, mode, rdev, mtime, +`--symlink-target`). This change implements that convention in s3fs as an +**opt-in** feature with three session knobs: **uid**, **gid**, and **umask**. + +### Constraints / assumptions + +- **Off by default.** When disabled, behavior is byte-for-byte what it is today. + `objgitd` itself never sets the option — the git protocol does not surface + POSIX attributes, so there's no win in enabling the feature for repo storage. + Other consumers of `internal/s3fs` (current or future) can opt in. +- **s3fs deals in numeric uid/gid.** The package does not resolve names→IDs or + IDs→names itself; it exposes optional helper functions and lets callers decide + whether/how to resolve. +- Scope = **create-time write + read** (uid/gid/mode/mtime). No chmod/chown + writeback (`billy.Change`) and no symlink/device support in this pass — those + are noted as follow-ups. + +## Approach + +### 1. New package `internal/s3fs/unixmeta` + +New file `internal/s3fs/unixmeta/unixmeta.go` implementing the doc verbatim: + +- `Attrs` struct, `PosixMode(os.FileMode) uint32`, `GoFileMode(uint32) os.FileMode`, + `Encode(Attrs) map[string]string`, `Decode(meta map[string]string, defaults Attrs) Attrs`. +- Provide optional, caller-invoked helpers (the package itself never calls them; + callers decide whether to resolve names): + - `LookupUID(name string) (uint32, error)` — `user.Lookup`, fall back to parsing + `name` as a decimal uint32. + - `LookupGID(name string) (uint32, error)` — `user.LookupGroup`, same fallback. + No reverse (uid/gid → name) resolution is provided. +- Table-driven tests `unixmeta_test.go`: PosixMode/GoFileMode round-trip across + file/dir/symlink/setuid/sticky; Encode→Decode round-trip; malformed-header + tolerance; missing-key-keeps-default. + +### 2. Opt-in config on `S3FS` (`internal/s3fs/filesystem.go`) + +Add a nil-able config (nil = disabled, preserving current behavior): + +```go +type unixMetaConfig struct { uid, gid uint32; umask os.FileMode } + +type S3FS struct { + client *storage.Client + bucket string + root, separator string + unixMeta *unixMetaConfig // nil => feature off + // ...existing tempfs fields... +} + +type Option func(*S3FS) +func WithUnixMetadata(uid, gid uint32, umask os.FileMode) Option + +func NewS3FS(client *storage.Client, bucket string, opts ...Option) (billy.Filesystem, error) +``` + +Variadic options keep the existing `cmd/objgitd` caller compiling unchanged. +`Chroot` must copy `unixMeta` onto the new `*S3FS` so chrooted views inherit the +session config. + +### 3. Write path (`internal/s3fs/file.go`) + +Thread `*unixMetaConfig` into `newS3WriteFile` and `newS3MultipartUploadFile` +(plumbed from `OpenFile` in `internal/s3fs/basic.go`). In each `Close()`: + +- If config is nil → unchanged (no `Metadata`). +- Else set `PutObjectInput.Metadata = unixmeta.Encode(unixmeta.Attrs{UID, GID, + Mode: 0o666 &^ umask, Mtime: time.Now()})`. (Multipart: `Metadata` on + `CreateMultipartUploadInput` at construction — `CompleteMultipartUpload` + cannot attach user metadata.) + +### 4. Read path (`internal/s3fs/basic.go`, `internal/s3fs/fileinfo.go`) + +- Extend `simpleFileInfo` with an optional `sys *FileStat` field; `Sys()` returns + the pointer when set, `nil` otherwise. `FileStat{UID, GID uint32}` lets + consumers read the raw numeric ownership and resolve to names themselves. +- Add `newFileInfoFromHead(name, head, cfg)` that, when `cfg != nil`, runs + `unixmeta.Decode(head.Metadata, defaults)` with defaults `{UID: cfg.uid, + GID: cfg.gid, Mode: 0o666 &^ cfg.umask, Mtime: head.LastModified}` and builds + a fully populated `simpleFileInfo`. When `cfg == nil`, keep the current + `0666` path by delegating to `newFileInfo`. +- Wire this into `Stat` (`internal/s3fs/basic.go`). `Lstat` already delegates + to `Stat`. +- **ReadDir** (`internal/s3fs/dir.go`): `ListObjectsV2` does not return user + metadata, so list entries keep default modes; full attributes come from + `Stat`. (`ls` stats entries for the long format.) Documented limitation; + avoids an N-Head fan-out. + +### 5. Consumer wiring + +No consumer in this repo currently opts in: + +- `cmd/objgitd` calls `s3fs.NewS3FS(client, *bucket)` with no options. Git + packfile/loose-object storage gains nothing from POSIX metadata, so adding + `-fs-*` flags to objgitd would be ceremony without payoff. The variadic + signature means the call site does not change. + +External callers (or a future objgit binary that exposes a POSIX-shaped surface) +opt in with: + +```go +s3fs.NewS3FS(client, bucket, s3fs.WithUnixMetadata(uid, gid, umask)) +``` + +and resolve any name strings to numeric IDs with +`unixmeta.LookupUID` / `unixmeta.LookupGID` before passing them in. + +## Files touched + +- `internal/s3fs/unixmeta/unixmeta.go` (new), `internal/s3fs/unixmeta/unixmeta_test.go` (new) +- `internal/s3fs/filesystem.go` — config + `Option` + `WithUnixMetadata` +- `internal/s3fs/chroot.go` — propagate `unixMeta` to the chrooted `*S3FS` +- `internal/s3fs/basic.go` — plumb config into `OpenFile`; decode in `Stat` +- `internal/s3fs/file.go` — encode metadata in write/multipart `Close` +- `internal/s3fs/fileinfo.go` — `FileStat`, optional `sys` field, decode constructor + +## Verification + +- `go test ./internal/s3fs/...` — unixmeta round-trip + tolerance tests pass. +- `go test ./...` — full suite, including the git-protocol tests, still passes + with the feature off (regression guard for the default path). +- `go build ./...` — `objgitd` still compiles. +- Manual (needs Tigris creds + `BUCKET`): a separate driver could open an + `s3fs` with `WithUnixMetadata`, write a file, then `HeadObject` (via + `tigris-objects` MCP / aws cli) and confirm `x-amz-meta-uid/gid/mode/mtime` + are present and correct; read it back and confirm `Stat().Mode()` reflects + `0666 &^ umask`. With the feature off, the same flow should show **no** + `x-amz-meta-*` headers. + +## Deferred (not in this change) + +- `billy.Change` (chmod/chown/chtimes) writeback via metadata-rewriting CopyObject. +- Symlink target / device-node (`rdev`) storage. +- Per-entry metadata in `ReadDir`. diff --git a/docs/reference/how-tigris-fs-unix-metadata.md b/docs/reference/how-tigris-fs-unix-metadata.md new file mode 100644 index 0000000..3047859 --- /dev/null +++ b/docs/reference/how-tigris-fs-unix-metadata.md @@ -0,0 +1,281 @@ +# How Unix metadata is stored on Tigris objects + +A POSIX filesystem mapped onto S3-compatible object storage needs somewhere to keep +Unix attributes (owner, group, permissions, timestamps, symlink targets) that S3 +itself does not natively model. The convention used here is to store them as +**S3 user-defined metadata** — a small set of `x-amz-meta-*` HTTP headers attached +to each object. + +This document describes the on-the-wire format and shows how to read and write it +with `aws-sdk-go-v2`. + +## The headers + +Every value is a string. Integers are decimal-encoded. + +| Header | Meaning | Value format | +|---|---|---| +| `x-amz-meta-uid` | Owner user ID | decimal `uint32` | +| `x-amz-meta-gid` | Owner group ID | decimal `uint32` | +| `x-amz-meta-mode` | POSIX file mode (type + permission bits) | decimal `uint32` | +| `x-amz-meta-rdev` | Device number (block/character devices only) | decimal `uint32` | +| `x-amz-meta-mtime` | Modification time | Unix seconds, decimal | +| `x-amz-meta---symlink-target` | Target path of a symlink | URL-percent-escaped string | + +The header names are lowercase. Headers are written only when the value differs +from the consumer's default; a missing header means "use the default" rather than +"value is zero." Typical defaults are the mounting user's UID/GID and `0o644` for +files / `0o755` for directories. + +The `--symlink-target` header really does have three dashes after `meta-`: the +attribute name is the literal string `--symlink-target`, and the S3 SDK prepends +the standard `x-amz-meta-` prefix. + +## Mode encoding + +The `mode` value is the standard POSIX `mode_t` integer: the low 9 bits encode +`rwxrwxrwx`, the next 3 encode setuid/setgid/sticky, and the file type lives in +the high bits. + +| Type | Octal mask | +|---|---| +| Regular file | `0o100000` | +| Directory | `0o040000` | +| Symbolic link | `0o120000` | +| Block device | `0o060000` | +| Character device | `0o020000` | +| FIFO | `0o010000` | +| Socket | `0o140000` | + +So `33188` (decimal) = `0o100644` = regular file with `rw-r--r--`. + +Go's `os.FileMode` uses a different layout (type bits at `1<<31` and friends), so +a conversion is required at the boundary. + +## Value escaping + +HTTP header values must be plain ASCII without control characters, and S3 +normalizes whitespace. To keep arbitrary bytes (UTF-8 paths, control characters, +`%`) round-tripping safely, values are percent-encoded before being put in the +header and decoded after reading. This is important for `--symlink-target`, +whose value is an arbitrary filesystem path. + +## Writing metadata + +The AWS SDK accepts user metadata as a `map[string]string` on +`PutObjectInput.Metadata`. The SDK prepends `x-amz-meta-` and lowercases the +keys, so you supply the bare attribute name. + +```go +package unixmeta + +import ( + "net/url" + "os" + "strconv" + "time" +) + +// PosixMode converts a Go os.FileMode to the POSIX mode_t integer +// stored in x-amz-meta-mode. +func PosixMode(m os.FileMode) uint32 { + out := uint32(m.Perm()) // low 9 permission bits + if m&os.ModeSetuid != 0 { + out |= 0o4000 + } + if m&os.ModeSetgid != 0 { + out |= 0o2000 + } + if m&os.ModeSticky != 0 { + out |= 0o1000 + } + switch { + case m&os.ModeDir != 0: + out |= 0o040000 + case m&os.ModeSymlink != 0: + out |= 0o120000 + case m&os.ModeDevice != 0 && m&os.ModeCharDevice != 0: + out |= 0o020000 + case m&os.ModeDevice != 0: + out |= 0o060000 + case m&os.ModeNamedPipe != 0: + out |= 0o010000 + case m&os.ModeSocket != 0: + out |= 0o140000 + default: + out |= 0o100000 + } + return out +} + +// Attrs is what a caller wants to record on an object. +type Attrs struct { + UID, GID uint32 + Mode os.FileMode + Rdev uint32 + Mtime time.Time + SymlinkTarget string // "" if not a symlink +} + +// Encode produces the user-metadata map for a PutObject call. +// Pass the result as PutObjectInput.Metadata. +func Encode(a Attrs) map[string]string { + mode := PosixMode(a.Mode) + m := map[string]string{ + "uid": strconv.FormatUint(uint64(a.UID), 10), + "gid": strconv.FormatUint(uint64(a.GID), 10), + "mode": strconv.FormatUint(uint64(mode), 10), + "mtime": strconv.FormatInt(a.Mtime.Unix(), 10), + } + if a.Mode&(os.ModeDevice|os.ModeCharDevice) != 0 { + m["rdev"] = strconv.FormatUint(uint64(a.Rdev), 10) + } + if a.SymlinkTarget != "" { + m["--symlink-target"] = url.QueryEscape(a.SymlinkTarget) + } + return m +} +``` + +Used at the call site: + +```go +_, err := client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String("mybucket"), + Key: aws.String("path/to/file"), + Body: body, + Metadata: unixmeta.Encode(unixmeta.Attrs{ + UID: 1000, + GID: 1000, + Mode: 0o644, + Mtime: time.Now(), + }), +}) +``` + +## Reading metadata + +`HeadObject` and `GetObject` both return user metadata in `Metadata +map[string]string`, with the `x-amz-meta-` prefix already stripped and keys +lowercased. + +```go +package unixmeta + +import ( + "net/url" + "os" + "strconv" + "time" +) + +// GoFileMode is the inverse of PosixMode. +func GoFileMode(p uint32) os.FileMode { + m := os.FileMode(p & 0o777) + if p&0o4000 != 0 { + m |= os.ModeSetuid + } + if p&0o2000 != 0 { + m |= os.ModeSetgid + } + if p&0o1000 != 0 { + m |= os.ModeSticky + } + switch p & 0o170000 { + case 0o040000: + m |= os.ModeDir + case 0o120000: + m |= os.ModeSymlink + case 0o020000: + m |= os.ModeDevice | os.ModeCharDevice + case 0o060000: + m |= os.ModeDevice + case 0o010000: + m |= os.ModeNamedPipe + case 0o140000: + m |= os.ModeSocket + case 0o100000: + // regular file: no extra bits + } + return m +} + +// Decode merges metadata from a HeadObject / GetObject response into defaults. +// Missing keys leave the corresponding field of `defaults` untouched, which is +// the behavior a POSIX filesystem usually wants: an object with no uid header +// inherits the mount's default uid, not zero. +func Decode(meta map[string]string, defaults Attrs) Attrs { + out := defaults + if s, ok := meta["uid"]; ok { + if v, err := strconv.ParseUint(s, 0, 32); err == nil { + out.UID = uint32(v) + } + } + if s, ok := meta["gid"]; ok { + if v, err := strconv.ParseUint(s, 0, 32); err == nil { + out.GID = uint32(v) + } + } + if s, ok := meta["mode"]; ok { + if v, err := strconv.ParseUint(s, 0, 32); err == nil { + out.Mode = GoFileMode(uint32(v)) + } + } + if s, ok := meta["rdev"]; ok { + if v, err := strconv.ParseUint(s, 0, 32); err == nil { + out.Rdev = uint32(v) + } + } + if s, ok := meta["mtime"]; ok { + if v, err := strconv.ParseInt(s, 0, 64); err == nil { + out.Mtime = time.Unix(v, 0) + } + } + if s, ok := meta["--symlink-target"]; ok { + if dec, err := url.QueryUnescape(s); err == nil { + out.SymlinkTarget = dec + } + } + return out +} +``` + +Used at the call site: + +```go +head, err := client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String("mybucket"), + Key: aws.String("path/to/file"), +}) +if err != nil { + return err +} + +attrs := unixmeta.Decode(head.Metadata, unixmeta.Attrs{ + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0o644, + // Mtime defaults to the object's S3 LastModified if mtime is missing: + Mtime: aws.ToTime(head.LastModified), +}) +``` + +## Parsing notes + +- `strconv.ParseUint(s, 0, ...)` with base `0` accepts decimal, `0o`-prefixed + octal, and `0x`-prefixed hex. Writers should emit decimal; readers should be + liberal. +- Treat malformed values as missing — parse errors fall through to the default + rather than failing the whole lookup. A single bad header should not make a + file unreadable. +- The `mode` header carries both the permission bits and the file-type bits. + When reapplying it to an in-memory inode, mask out the type bits if you only + want to update permissions (for `chmod`), and mask out the permission bits if + you only want to update the type (rare — typically set once at creation). + +## Directories and zero-byte objects + +Directories are zero-byte S3 objects whose key ends in `/`. They carry the same +metadata headers as regular files, with `mode` containing the directory type bit +(`0o040000`). Symlinks are likewise zero-byte objects; the link target lives +entirely in the `--symlink-target` header, not in the object body. diff --git a/internal/s3fs/basic.go b/internal/s3fs/basic.go index 6f3402c..d0c7ca0 100644 --- a/internal/s3fs/basic.go +++ b/internal/s3fs/basic.go @@ -13,7 +13,6 @@ import ( "strings" "time" - "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/smithy-go" "github.com/go-git/go-billy/v6" @@ -110,10 +109,10 @@ func (fs3 *S3FS) OpenFile(filename string, flag int, perm os.FileMode) (billy.Fi return nil, &os.PathError{Op: "open", Path: filename, Err: fs.ErrNotExist} case O_WRONLY: - return newS3WriteFile(fs3.client, fs3.bucket, key, filename) + return newS3WriteFile(fs3.client, fs3.bucket, key, filename, fs3.unixMeta) case O_WRMULTIPART: - return newS3MultipartUploadFile(fs3.client, fs3.bucket, key, filename) + return newS3MultipartUploadFile(fs3.client, fs3.bucket, key, filename, fs3.unixMeta) default: return nil, errors.New("unsupported open flag") @@ -140,11 +139,7 @@ func (fs3 *S3FS) Stat(filename string) (os.FileInfo, error) { Key: &key, }) if err == nil { - return newFileInfo( - path.Base(key), - aws.ToInt64(head.ContentLength), - aws.ToTime(head.LastModified), - ), nil + return newFileInfoFromHead(path.Base(key), head, fs3.unixMeta), nil } var apiErr smithy.APIError diff --git a/internal/s3fs/chroot.go b/internal/s3fs/chroot.go index c081c7b..5c7a601 100644 --- a/internal/s3fs/chroot.go +++ b/internal/s3fs/chroot.go @@ -23,6 +23,7 @@ func (fs3 *S3FS) Chroot(path string) (billy.Filesystem, error) { bucket: fs3.bucket, root: p, separator: fs3.separator, + unixMeta: fs3.unixMeta, temps: make(map[string]*tempBuffer), } return nfs, nil diff --git a/internal/s3fs/file.go b/internal/s3fs/file.go index e4a5685..54433c2 100644 --- a/internal/s3fs/file.go +++ b/internal/s3fs/file.go @@ -15,8 +15,24 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/tigrisdata/storage-go" "go.uber.org/atomic" + "tangled.org/xeiaso.net/objgit/internal/s3fs/unixmeta" ) +// newFileMetadata returns the x-amz-meta-* map to attach to a newly written +// object, or nil when the Unix-metadata feature is disabled. New files take the +// session's default owner and a mode of 0666 masked by the session umask. +func newFileMetadata(cfg *unixMetaConfig) map[string]string { + if cfg == nil { + return nil + } + return unixmeta.Encode(unixmeta.Attrs{ + UID: cfg.uid, + GID: cfg.gid, + Mode: 0o666 &^ cfg.umask, + Mtime: time.Now(), + }) +} + const ( ModeMultipartUpload os.FileMode = fs.ModePerm + 1 // Custom os.FileMode for S3 multipart upload ) @@ -159,23 +175,25 @@ func (f *s3ReadFile) Stat() (fs.FileInfo, error) { // Upon creation, a buffer is created to store the file contents. Upon close, // the file is uploaded to S3. type s3WriteFile struct { - client *storage.Client // s3 skd client - bucket string // S3 bucket name - key string // File object's key in S3 - name string // Root-relative path as presented to Open - closed bool // Is the file closed? - buf *bytes.Buffer // Buffer for storing the file before it's uploaded + client *storage.Client // s3 skd client + bucket string // S3 bucket name + key string // File object's key in S3 + name string // Root-relative path as presented to Open + closed bool // Is the file closed? + buf *bytes.Buffer // Buffer for storing the file before it's uploaded + unixMeta *unixMetaConfig // optional POSIX attribute defaults (nil = disabled) } // newS3WriteFile creates a new s3WriteFile. key is the full S3 object key; name // is the root-relative path the caller passed to Open (returned by Name). -func newS3WriteFile(client *storage.Client, bucket, key, name string) (*s3WriteFile, error) { +func newS3WriteFile(client *storage.Client, bucket, key, name string, cfg *unixMetaConfig) (*s3WriteFile, error) { return &s3WriteFile{ - client: client, - bucket: bucket, - key: key, - name: name, - buf: bytes.NewBuffer(nil), + client: client, + bucket: bucket, + key: key, + name: name, + buf: bytes.NewBuffer(nil), + unixMeta: cfg, }, nil } @@ -229,9 +247,10 @@ func (f *s3WriteFile) Close() error { // Run the GetObject operation // TODO: Currently `res` is not used. Should it be? _, err := f.client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &f.bucket, - Key: &f.key, - Body: body, + Bucket: &f.bucket, + Key: &f.key, + Body: body, + Metadata: newFileMetadata(f.unixMeta), }) if err != nil { return fmt.Errorf("unable to perform GetObject operation: %w", err) @@ -273,17 +292,19 @@ type s3MultipartUploadFile struct { // newS3MultipartUploadFile creates a new s3MultipartUploadFile. key is the full // S3 object key; name is the root-relative path passed to Open. -func newS3MultipartUploadFile(client *storage.Client, bucket, key, name string) (*s3MultipartUploadFile, error) { +func newS3MultipartUploadFile(client *storage.Client, bucket, key, name string, cfg *unixMetaConfig) (*s3MultipartUploadFile, error) { // TODO: Check if the file exists // ... // Create the context ctx := context.TODO() // TODO: How can user-supplied contexts be supported? - // Run the GetObject operation + // Run the GetObject operation. POSIX attributes (if enabled) must be set + // now: CompleteMultipartUpload cannot attach user metadata. res, err := client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &key, + Bucket: &bucket, + Key: &key, + Metadata: newFileMetadata(cfg), }) if err != nil { return nil, fmt.Errorf("unable to create multipart upload: %w", err) diff --git a/internal/s3fs/fileinfo.go b/internal/s3fs/fileinfo.go index 7fb243a..9171972 100644 --- a/internal/s3fs/fileinfo.go +++ b/internal/s3fs/fileinfo.go @@ -5,15 +5,25 @@ import ( "os" "time" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" + "tangled.org/xeiaso.net/objgit/internal/s3fs/unixmeta" ) +// FileStat is the value returned by simpleFileInfo.Sys() when the +// Unix-metadata feature is enabled. It carries the raw numeric owner and group +// so consumers can resolve them to names however they like. +type FileStat struct { + UID, GID uint32 +} + // simpleFileInfo implements os.FileInfo type simpleFileInfo struct { name string size int64 mode os.FileMode modTime time.Time + sys *FileStat } func newFileInfo(name string, size int64, modTime time.Time) os.FileInfo { @@ -33,11 +43,43 @@ func newDirInfo(name string) os.FileInfo { } } -func (fi simpleFileInfo) Name() string { return fi.name } -func (fi simpleFileInfo) Size() int64 { return fi.size } -func (fi simpleFileInfo) Mode() os.FileMode { return fi.mode } -func (fi simpleFileInfo) IsDir() bool { return fi.mode.IsDir() } -func (fi simpleFileInfo) Sys() interface{} { return nil } +// newFileInfoFromHead builds a FileInfo from a HeadObject response. When cfg is +// nil the Unix-metadata feature is off and the result matches newFileInfo; +// otherwise the x-amz-meta-* attributes are decoded, falling back to the +// session defaults for any header that is missing. +func newFileInfoFromHead(name string, head *s3.HeadObjectOutput, cfg *unixMetaConfig) os.FileInfo { + size := aws.ToInt64(head.ContentLength) + modTime := aws.ToTime(head.LastModified) + if cfg == nil { + return newFileInfo(name, size, modTime) + } + + attrs := unixmeta.Decode(head.Metadata, unixmeta.Attrs{ + UID: cfg.uid, + GID: cfg.gid, + Mode: 0o666 &^ cfg.umask, + Mtime: modTime, + }) + + return simpleFileInfo{ + name: name, + size: size, + mode: attrs.Mode, + modTime: attrs.Mtime, + sys: &FileStat{UID: attrs.UID, GID: attrs.GID}, + } +} + +func (fi simpleFileInfo) Name() string { return fi.name } +func (fi simpleFileInfo) Size() int64 { return fi.size } +func (fi simpleFileInfo) Mode() os.FileMode { return fi.mode } +func (fi simpleFileInfo) IsDir() bool { return fi.mode.IsDir() } +func (fi simpleFileInfo) Sys() any { + if fi.sys == nil { + return nil + } + return fi.sys +} func (fi simpleFileInfo) ModTime() time.Time { return fi.modTime } type enrichedFileInfo struct { diff --git a/internal/s3fs/filesystem.go b/internal/s3fs/filesystem.go index 1db7b48..74d323c 100644 --- a/internal/s3fs/filesystem.go +++ b/internal/s3fs/filesystem.go @@ -2,6 +2,7 @@ package s3fs import ( "fmt" + "os" "path" "strings" "sync" @@ -14,11 +15,20 @@ const ( DefaultSeparator = "/" ) +// unixMetaConfig holds the session defaults used when the optional Unix-metadata +// feature is enabled. A nil *unixMetaConfig means the feature is off and the +// filesystem behaves as if no POSIX attributes exist. +type unixMetaConfig struct { + uid, gid uint32 + umask os.FileMode +} + type S3FS struct { client *storage.Client bucket string root string separator string + unixMeta *unixMetaConfig // temps holds TempFile-backed buffers keyed by canonical S3 key, so a // subsequent Open of the same path returns a reader over the same bytes @@ -27,19 +37,37 @@ type S3FS struct { temps map[string]*tempBuffer } +// Option configures an S3FS at construction time. +type Option func(*S3FS) + +// WithUnixMetadata enables storing and reading POSIX file attributes as S3 user +// metadata (see the unixmeta package). uid and gid are the numeric owner/group +// recorded on newly written objects; callers resolve any names to numbers +// themselves. umask is applied to the default mode (0666 for files) when +// writing. Without this option the filesystem stores no attributes. +func WithUnixMetadata(uid, gid uint32, umask os.FileMode) Option { + return func(fs3 *S3FS) { + fs3.unixMeta = &unixMetaConfig{uid: uid, gid: gid, umask: umask} + } +} + // NewS3FS creates a new S3FS Filesystem. -func NewS3FS(client *storage.Client, bucket string) (billy.Filesystem, error) { +func NewS3FS(client *storage.Client, bucket string, opts ...Option) (billy.Filesystem, error) { // Check for a non-nil client if client == nil { return nil, fmt.Errorf("s3 client cannot be nil") } - return &S3FS{ + fs3 := &S3FS{ client: client, bucket: bucket, root: "", separator: DefaultSeparator, temps: make(map[string]*tempBuffer), - }, nil + } + for _, opt := range opts { + opt(fs3) + } + return fs3, nil } // Capabilities returns the filesystem capabilities. diff --git a/internal/s3fs/unixmeta/unixmeta.go b/internal/s3fs/unixmeta/unixmeta.go new file mode 100644 index 0000000..9ce2816 --- /dev/null +++ b/internal/s3fs/unixmeta/unixmeta.go @@ -0,0 +1,207 @@ +// Package unixmeta encodes and decodes POSIX file attributes (owner, group, +// permissions, timestamps) as S3 user-defined metadata (x-amz-meta-* headers). +// +// S3 does not natively model Unix attributes, so a POSIX filesystem layered on +// top of object storage keeps them in a small set of string-valued metadata +// headers. The on-the-wire format is documented in +// docs/reference/how-tigris-fs-unix-metadata.md. +package unixmeta + +import ( + "net/url" + "os" + "os/user" + "strconv" + "time" +) + +// Metadata keys (without the x-amz-meta- prefix the S3 SDK prepends). +const ( + keyUID = "uid" + keyGID = "gid" + keyMode = "mode" + keyRdev = "rdev" + keyMtime = "mtime" + keySymlinkTarget = "--symlink-target" +) + +// POSIX file-type mask and the bits stored in the high part of mode_t. +const ( + modeTypeMask = 0o170000 + modeRegular = 0o100000 + modeDir = 0o040000 + modeSymlink = 0o120000 + modeBlock = 0o060000 + modeChar = 0o020000 + modeFIFO = 0o010000 + modeSocket = 0o140000 +) + +// Attrs is the set of POSIX attributes recorded on an object. +type Attrs struct { + UID, GID uint32 + Mode os.FileMode + Rdev uint32 + Mtime time.Time + SymlinkTarget string // "" if not a symlink +} + +// PosixMode converts a Go os.FileMode to the POSIX mode_t integer stored in +// x-amz-meta-mode. +func PosixMode(m os.FileMode) uint32 { + out := uint32(m.Perm()) // low 9 permission bits + if m&os.ModeSetuid != 0 { + out |= 0o4000 + } + if m&os.ModeSetgid != 0 { + out |= 0o2000 + } + if m&os.ModeSticky != 0 { + out |= 0o1000 + } + switch { + case m&os.ModeDir != 0: + out |= modeDir + case m&os.ModeSymlink != 0: + out |= modeSymlink + case m&os.ModeDevice != 0 && m&os.ModeCharDevice != 0: + out |= modeChar + case m&os.ModeDevice != 0: + out |= modeBlock + case m&os.ModeNamedPipe != 0: + out |= modeFIFO + case m&os.ModeSocket != 0: + out |= modeSocket + default: + out |= modeRegular + } + return out +} + +// GoFileMode is the inverse of PosixMode. +func GoFileMode(p uint32) os.FileMode { + m := os.FileMode(p & 0o777) + if p&0o4000 != 0 { + m |= os.ModeSetuid + } + if p&0o2000 != 0 { + m |= os.ModeSetgid + } + if p&0o1000 != 0 { + m |= os.ModeSticky + } + switch p & modeTypeMask { + case modeDir: + m |= os.ModeDir + case modeSymlink: + m |= os.ModeSymlink + case modeChar: + m |= os.ModeDevice | os.ModeCharDevice + case modeBlock: + m |= os.ModeDevice + case modeFIFO: + m |= os.ModeNamedPipe + case modeSocket: + m |= os.ModeSocket + case modeRegular: + // regular file: no extra bits + } + return m +} + +// Encode produces the user-metadata map for a PutObject call. Pass the result +// as PutObjectInput.Metadata; the S3 SDK prepends x-amz-meta- and lowercases +// the keys. +func Encode(a Attrs) map[string]string { + mode := PosixMode(a.Mode) + m := map[string]string{ + keyUID: strconv.FormatUint(uint64(a.UID), 10), + keyGID: strconv.FormatUint(uint64(a.GID), 10), + keyMode: strconv.FormatUint(uint64(mode), 10), + keyMtime: strconv.FormatInt(a.Mtime.Unix(), 10), + } + if a.Mode&(os.ModeDevice|os.ModeCharDevice) != 0 { + m[keyRdev] = strconv.FormatUint(uint64(a.Rdev), 10) + } + if a.SymlinkTarget != "" { + m[keySymlinkTarget] = url.QueryEscape(a.SymlinkTarget) + } + return m +} + +// Decode merges metadata from a HeadObject / GetObject response into defaults. +// Missing keys leave the corresponding field of defaults untouched, which is +// the behavior a POSIX filesystem usually wants: an object with no uid header +// inherits the mount's default uid, not zero. Malformed values are treated as +// missing so a single bad header doesn't make a file unreadable. +func Decode(meta map[string]string, defaults Attrs) Attrs { + out := defaults + if s, ok := meta[keyUID]; ok { + if v, err := strconv.ParseUint(s, 0, 32); err == nil { + out.UID = uint32(v) + } + } + if s, ok := meta[keyGID]; ok { + if v, err := strconv.ParseUint(s, 0, 32); err == nil { + out.GID = uint32(v) + } + } + if s, ok := meta[keyMode]; ok { + if v, err := strconv.ParseUint(s, 0, 32); err == nil { + out.Mode = GoFileMode(uint32(v)) + } + } + if s, ok := meta[keyRdev]; ok { + if v, err := strconv.ParseUint(s, 0, 32); err == nil { + out.Rdev = uint32(v) + } + } + if s, ok := meta[keyMtime]; ok { + if v, err := strconv.ParseInt(s, 0, 64); err == nil { + out.Mtime = time.Unix(v, 0) + } + } + if s, ok := meta[keySymlinkTarget]; ok { + if dec, err := url.QueryUnescape(s); err == nil { + out.SymlinkTarget = dec + } + } + return out +} + +// LookupUID resolves a user name to a numeric UID. It first consults the host +// passwd database via os/user; if the name is not found there it is parsed as a +// decimal UID. This lets callers pass either "alice" or "1000" and works in +// containers that lack a matching passwd entry. The package never calls this +// itself — callers decide whether to resolve names. +func LookupUID(name string) (uint32, error) { + if u, err := user.Lookup(name); err == nil { + v, perr := strconv.ParseUint(u.Uid, 10, 32) + if perr != nil { + return 0, perr + } + return uint32(v), nil + } + v, err := strconv.ParseUint(name, 10, 32) + if err != nil { + return 0, err + } + return uint32(v), nil +} + +// LookupGID resolves a group name to a numeric GID, with the same name-or-number +// semantics as LookupUID. +func LookupGID(name string) (uint32, error) { + if g, err := user.LookupGroup(name); err == nil { + v, perr := strconv.ParseUint(g.Gid, 10, 32) + if perr != nil { + return 0, perr + } + return uint32(v), nil + } + v, err := strconv.ParseUint(name, 10, 32) + if err != nil { + return 0, err + } + return uint32(v), nil +} diff --git a/internal/s3fs/unixmeta/unixmeta_test.go b/internal/s3fs/unixmeta/unixmeta_test.go new file mode 100644 index 0000000..e89f38c --- /dev/null +++ b/internal/s3fs/unixmeta/unixmeta_test.go @@ -0,0 +1,132 @@ +package unixmeta + +import ( + "os" + "testing" + "time" +) + +func TestModeRoundTrip(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + name string + mode os.FileMode + }{ + {name: "regular file rw-r--r--", mode: 0o644}, + {name: "regular file rwxr-xr-x", mode: 0o755}, + {name: "directory", mode: os.ModeDir | 0o755}, + {name: "symlink", mode: os.ModeSymlink | 0o777}, + {name: "block device", mode: os.ModeDevice | 0o660}, + {name: "char device", mode: os.ModeDevice | os.ModeCharDevice | 0o620}, + {name: "fifo", mode: os.ModeNamedPipe | 0o644}, + {name: "socket", mode: os.ModeSocket | 0o755}, + {name: "setuid", mode: os.ModeSetuid | 0o755}, + {name: "setgid", mode: os.ModeSetgid | 0o755}, + {name: "sticky dir", mode: os.ModeDir | os.ModeSticky | 0o777}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := GoFileMode(PosixMode(tt.mode)) + if got != tt.mode { + t.Logf("want: %v (%#o)", tt.mode, uint32(tt.mode)) + t.Logf("got: %v (%#o)", got, uint32(got)) + t.Error("mode did not survive round trip") + } + }) + } +} + +func TestPosixModeKnownValue(t *testing.T) { + t.Parallel() + + // 0o100644 == 33188: regular file with rw-r--r--, per the reference doc. + if got := PosixMode(0o644); got != 0o100644 { + t.Errorf("PosixMode(0o644) = %#o, want %#o", got, 0o100644) + } +} + +func TestEncodeDecodeRoundTrip(t *testing.T) { + t.Parallel() + + mtime := time.Unix(1716700000, 0) + + for _, tt := range []struct { + name string + in Attrs + }{ + { + name: "regular file", + in: Attrs{UID: 1000, GID: 1000, Mode: 0o644, Mtime: mtime}, + }, + { + name: "directory", + in: Attrs{UID: 0, GID: 0, Mode: os.ModeDir | 0o755, Mtime: mtime}, + }, + { + name: "char device with rdev", + in: Attrs{UID: 0, GID: 5, Mode: os.ModeDevice | os.ModeCharDevice | 0o620, Rdev: 1280, Mtime: mtime}, + }, + { + name: "symlink with awkward target", + in: Attrs{UID: 1000, GID: 1000, Mode: os.ModeSymlink | 0o777, Mtime: mtime, SymlinkTarget: "/etc/has spaces/and%percent"}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := Decode(Encode(tt.in), Attrs{}) + if got.UID != tt.in.UID || got.GID != tt.in.GID { + t.Errorf("uid/gid: want %d/%d, got %d/%d", tt.in.UID, tt.in.GID, got.UID, got.GID) + } + if got.Mode != tt.in.Mode { + t.Errorf("mode: want %v, got %v", tt.in.Mode, got.Mode) + } + if !got.Mtime.Equal(tt.in.Mtime) { + t.Errorf("mtime: want %v, got %v", tt.in.Mtime, got.Mtime) + } + if got.SymlinkTarget != tt.in.SymlinkTarget { + t.Errorf("symlink target: want %q, got %q", tt.in.SymlinkTarget, got.SymlinkTarget) + } + if tt.in.Rdev != 0 && got.Rdev != tt.in.Rdev { + t.Errorf("rdev: want %d, got %d", tt.in.Rdev, got.Rdev) + } + }) + } +} + +func TestDecodeMissingKeysKeepDefaults(t *testing.T) { + t.Parallel() + + defaults := Attrs{UID: 501, GID: 20, Mode: 0o644, Mtime: time.Unix(42, 0)} + got := Decode(map[string]string{}, defaults) + if got != defaults { + t.Logf("want: %+v", defaults) + t.Logf("got: %+v", got) + t.Error("empty metadata should leave defaults untouched") + } +} + +func TestDecodeMalformedTreatedAsMissing(t *testing.T) { + t.Parallel() + + defaults := Attrs{UID: 501, GID: 20, Mode: 0o644, Mtime: time.Unix(42, 0)} + meta := map[string]string{ + "uid": "not-a-number", + "gid": "99", + "mode": "garbage", + "mtime": "also-bad", + } + got := Decode(meta, defaults) + if got.UID != defaults.UID { + t.Errorf("malformed uid should fall back to default: want %d, got %d", defaults.UID, got.UID) + } + if got.GID != 99 { + t.Errorf("valid gid should be parsed: want 99, got %d", got.GID) + } + if got.Mode != defaults.Mode { + t.Errorf("malformed mode should fall back to default: want %v, got %v", defaults.Mode, got.Mode) + } + if !got.Mtime.Equal(defaults.Mtime) { + t.Errorf("malformed mtime should fall back to default: want %v, got %v", defaults.Mtime, got.Mtime) + } +} -- 2.51.2