diff --git a/docs/plans/s3fs-unix-metadata.md b/docs/plans/s3fs-unix-metadata.md new file mode 100644 index 0000000..29f5fa6 --- /dev/null +++ b/docs/plans/s3fs-unix-metadata.md @@ -0,0 +1,145 @@ +# Plan: Optional Unix-permission metadata in s3fs + +## Context + +`internal/s3fs/` is a go-billy filesystem mapped onto Tigris (S3) storage, +consumed today by `cmd/objgitd` to back git repositories. It never reads or +writes POSIX attributes: every file reports mode `0666`, directories `ModeDir`, +uid/gid are absent, and `PutObject` carries no user metadata +(`internal/s3fs/fileinfo.go`, `internal/s3fs/file.go`). + +`docs/reference/how-tigris-fs-unix-metadata.md` defines a convention for storing +Unix attributes as `x-amz-meta-*` headers (uid, gid, mode, rdev, mtime, +`--symlink-target`). This change implements that convention in s3fs as an +**opt-in** feature with three session knobs: **uid**, **gid**, and **umask**. + +### Constraints / assumptions + +- **Off by default.** When disabled, behavior is byte-for-byte what it is today. + `objgitd` itself never sets the option — the git protocol does not surface + POSIX attributes, so there's no win in enabling the feature for repo storage. + Other consumers of `internal/s3fs` (current or future) can opt in. +- **s3fs deals in numeric uid/gid.** The package does not resolve names→IDs or + IDs→names itself; it exposes optional helper functions and lets callers decide + whether/how to resolve. +- Scope = **create-time write + read** (uid/gid/mode/mtime). No chmod/chown + writeback (`billy.Change`) and no symlink/device support in this pass — those + are noted as follow-ups. + +## Approach + +### 1. New package `internal/s3fs/unixmeta` + +New file `internal/s3fs/unixmeta/unixmeta.go` implementing the doc verbatim: + +- `Attrs` struct, `PosixMode(os.FileMode) uint32`, `GoFileMode(uint32) os.FileMode`, + `Encode(Attrs) map[string]string`, `Decode(meta map[string]string, defaults Attrs) Attrs`. +- Provide optional, caller-invoked helpers (the package itself never calls them; + callers decide whether to resolve names): + - `LookupUID(name string) (uint32, error)` — `user.Lookup`, fall back to parsing + `name` as a decimal uint32. + - `LookupGID(name string) (uint32, error)` — `user.LookupGroup`, same fallback. + No reverse (uid/gid → name) resolution is provided. +- Table-driven tests `unixmeta_test.go`: PosixMode/GoFileMode round-trip across + file/dir/symlink/setuid/sticky; Encode→Decode round-trip; malformed-header + tolerance; missing-key-keeps-default. + +### 2. Opt-in config on `S3FS` (`internal/s3fs/filesystem.go`) + +Add a nil-able config (nil = disabled, preserving current behavior): + +```go +type unixMetaConfig struct { uid, gid uint32; umask os.FileMode } + +type S3FS struct { + client *storage.Client + bucket string + root, separator string + unixMeta *unixMetaConfig // nil => feature off + // ...existing tempfs fields... +} + +type Option func(*S3FS) +func WithUnixMetadata(uid, gid uint32, umask os.FileMode) Option + +func NewS3FS(client *storage.Client, bucket string, opts ...Option) (billy.Filesystem, error) +``` + +Variadic options keep the existing `cmd/objgitd` caller compiling unchanged. +`Chroot` must copy `unixMeta` onto the new `*S3FS` so chrooted views inherit the +session config. + +### 3. Write path (`internal/s3fs/file.go`) + +Thread `*unixMetaConfig` into `newS3WriteFile` and `newS3MultipartUploadFile` +(plumbed from `OpenFile` in `internal/s3fs/basic.go`). In each `Close()`: + +- If config is nil → unchanged (no `Metadata`). +- Else set `PutObjectInput.Metadata = unixmeta.Encode(unixmeta.Attrs{UID, GID, + Mode: 0o666 &^ umask, Mtime: time.Now()})`. (Multipart: `Metadata` on + `CreateMultipartUploadInput` at construction — `CompleteMultipartUpload` + cannot attach user metadata.) + +### 4. Read path (`internal/s3fs/basic.go`, `internal/s3fs/fileinfo.go`) + +- Extend `simpleFileInfo` with an optional `sys *FileStat` field; `Sys()` returns + the pointer when set, `nil` otherwise. `FileStat{UID, GID uint32}` lets + consumers read the raw numeric ownership and resolve to names themselves. +- Add `newFileInfoFromHead(name, head, cfg)` that, when `cfg != nil`, runs + `unixmeta.Decode(head.Metadata, defaults)` with defaults `{UID: cfg.uid, + GID: cfg.gid, Mode: 0o666 &^ cfg.umask, Mtime: head.LastModified}` and builds + a fully populated `simpleFileInfo`. When `cfg == nil`, keep the current + `0666` path by delegating to `newFileInfo`. +- Wire this into `Stat` (`internal/s3fs/basic.go`). `Lstat` already delegates + to `Stat`. +- **ReadDir** (`internal/s3fs/dir.go`): `ListObjectsV2` does not return user + metadata, so list entries keep default modes; full attributes come from + `Stat`. (`ls` stats entries for the long format.) Documented limitation; + avoids an N-Head fan-out. + +### 5. Consumer wiring + +No consumer in this repo currently opts in: + +- `cmd/objgitd` calls `s3fs.NewS3FS(client, *bucket)` with no options. Git + packfile/loose-object storage gains nothing from POSIX metadata, so adding + `-fs-*` flags to objgitd would be ceremony without payoff. The variadic + signature means the call site does not change. + +External callers (or a future objgit binary that exposes a POSIX-shaped surface) +opt in with: + +```go +s3fs.NewS3FS(client, bucket, s3fs.WithUnixMetadata(uid, gid, umask)) +``` + +and resolve any name strings to numeric IDs with +`unixmeta.LookupUID` / `unixmeta.LookupGID` before passing them in. + +## Files touched + +- `internal/s3fs/unixmeta/unixmeta.go` (new), `internal/s3fs/unixmeta/unixmeta_test.go` (new) +- `internal/s3fs/filesystem.go` — config + `Option` + `WithUnixMetadata` +- `internal/s3fs/chroot.go` — propagate `unixMeta` to the chrooted `*S3FS` +- `internal/s3fs/basic.go` — plumb config into `OpenFile`; decode in `Stat` +- `internal/s3fs/file.go` — encode metadata in write/multipart `Close` +- `internal/s3fs/fileinfo.go` — `FileStat`, optional `sys` field, decode constructor + +## Verification + +- `go test ./internal/s3fs/...` — unixmeta round-trip + tolerance tests pass. +- `go test ./...` — full suite, including the git-protocol tests, still passes + with the feature off (regression guard for the default path). +- `go build ./...` — `objgitd` still compiles. +- Manual (needs Tigris creds + `BUCKET`): a separate driver could open an + `s3fs` with `WithUnixMetadata`, write a file, then `HeadObject` (via + `tigris-objects` MCP / aws cli) and confirm `x-amz-meta-uid/gid/mode/mtime` + are present and correct; read it back and confirm `Stat().Mode()` reflects + `0666 &^ umask`. With the feature off, the same flow should show **no** + `x-amz-meta-*` headers. + +## Deferred (not in this change) + +- `billy.Change` (chmod/chown/chtimes) writeback via metadata-rewriting CopyObject. +- Symlink target / device-node (`rdev`) storage. +- Per-entry metadata in `ReadDir`. diff --git a/docs/reference/how-tigris-fs-unix-metadata.md b/docs/reference/how-tigris-fs-unix-metadata.md new file mode 100644 index 0000000..3047859 --- /dev/null +++ b/docs/reference/how-tigris-fs-unix-metadata.md @@ -0,0 +1,281 @@ +# How Unix metadata is stored on Tigris objects + +A POSIX filesystem mapped onto S3-compatible object storage needs somewhere to keep +Unix attributes (owner, group, permissions, timestamps, symlink targets) that S3 +itself does not natively model. The convention used here is to store them as +**S3 user-defined metadata** — a small set of `x-amz-meta-*` HTTP headers attached +to each object. + +This document describes the on-the-wire format and shows how to read and write it +with `aws-sdk-go-v2`. + +## The headers + +Every value is a string. Integers are decimal-encoded. + +| Header | Meaning | Value format | +|---|---|---| +| `x-amz-meta-uid` | Owner user ID | decimal `uint32` | +| `x-amz-meta-gid` | Owner group ID | decimal `uint32` | +| `x-amz-meta-mode` | POSIX file mode (type + permission bits) | decimal `uint32` | +| `x-amz-meta-rdev` | Device number (block/character devices only) | decimal `uint32` | +| `x-amz-meta-mtime` | Modification time | Unix seconds, decimal | +| `x-amz-meta---symlink-target` | Target path of a symlink | URL-percent-escaped string | + +The header names are lowercase. Headers are written only when the value differs +from the consumer's default; a missing header means "use the default" rather than +"value is zero." Typical defaults are the mounting user's UID/GID and `0o644` for +files / `0o755` for directories. + +The `--symlink-target` header really does have three dashes after `meta-`: the +attribute name is the literal string `--symlink-target`, and the S3 SDK prepends +the standard `x-amz-meta-` prefix. + +## Mode encoding + +The `mode` value is the standard POSIX `mode_t` integer: the low 9 bits encode +`rwxrwxrwx`, the next 3 encode setuid/setgid/sticky, and the file type lives in +the high bits. + +| Type | Octal mask | +|---|---| +| Regular file | `0o100000` | +| Directory | `0o040000` | +| Symbolic link | `0o120000` | +| Block device | `0o060000` | +| Character device | `0o020000` | +| FIFO | `0o010000` | +| Socket | `0o140000` | + +So `33188` (decimal) = `0o100644` = regular file with `rw-r--r--`. + +Go's `os.FileMode` uses a different layout (type bits at `1<<31` and friends), so +a conversion is required at the boundary. + +## Value escaping + +HTTP header values must be plain ASCII without control characters, and S3 +normalizes whitespace. To keep arbitrary bytes (UTF-8 paths, control characters, +`%`) round-tripping safely, values are percent-encoded before being put in the +header and decoded after reading. This is important for `--symlink-target`, +whose value is an arbitrary filesystem path. + +## Writing metadata + +The AWS SDK accepts user metadata as a `map[string]string` on +`PutObjectInput.Metadata`. The SDK prepends `x-amz-meta-` and lowercases the +keys, so you supply the bare attribute name. + +```go +package unixmeta + +import ( + "net/url" + "os" + "strconv" + "time" +) + +// PosixMode converts a Go os.FileMode to the POSIX mode_t integer +// stored in x-amz-meta-mode. +func PosixMode(m os.FileMode) uint32 { + out := uint32(m.Perm()) // low 9 permission bits + if m&os.ModeSetuid != 0 { + out |= 0o4000 + } + if m&os.ModeSetgid != 0 { + out |= 0o2000 + } + if m&os.ModeSticky != 0 { + out |= 0o1000 + } + switch { + case m&os.ModeDir != 0: + out |= 0o040000 + case m&os.ModeSymlink != 0: + out |= 0o120000 + case m&os.ModeDevice != 0 && m&os.ModeCharDevice != 0: + out |= 0o020000 + case m&os.ModeDevice != 0: + out |= 0o060000 + case m&os.ModeNamedPipe != 0: + out |= 0o010000 + case m&os.ModeSocket != 0: + out |= 0o140000 + default: + out |= 0o100000 + } + return out +} + +// Attrs is what a caller wants to record on an object. +type Attrs struct { + UID, GID uint32 + Mode os.FileMode + Rdev uint32 + Mtime time.Time + SymlinkTarget string // "" if not a symlink +} + +// Encode produces the user-metadata map for a PutObject call. +// Pass the result as PutObjectInput.Metadata. +func Encode(a Attrs) map[string]string { + mode := PosixMode(a.Mode) + m := map[string]string{ + "uid": strconv.FormatUint(uint64(a.UID), 10), + "gid": strconv.FormatUint(uint64(a.GID), 10), + "mode": strconv.FormatUint(uint64(mode), 10), + "mtime": strconv.FormatInt(a.Mtime.Unix(), 10), + } + if a.Mode&(os.ModeDevice|os.ModeCharDevice) != 0 { + m["rdev"] = strconv.FormatUint(uint64(a.Rdev), 10) + } + if a.SymlinkTarget != "" { + m["--symlink-target"] = url.QueryEscape(a.SymlinkTarget) + } + return m +} +``` + +Used at the call site: + +```go +_, err := client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String("mybucket"), + Key: aws.String("path/to/file"), + Body: body, + Metadata: unixmeta.Encode(unixmeta.Attrs{ + UID: 1000, + GID: 1000, + Mode: 0o644, + Mtime: time.Now(), + }), +}) +``` + +## Reading metadata + +`HeadObject` and `GetObject` both return user metadata in `Metadata +map[string]string`, with the `x-amz-meta-` prefix already stripped and keys +lowercased. + +```go +package unixmeta + +import ( + "net/url" + "os" + "strconv" + "time" +) + +// GoFileMode is the inverse of PosixMode. +func GoFileMode(p uint32) os.FileMode { + m := os.FileMode(p & 0o777) + if p&0o4000 != 0 { + m |= os.ModeSetuid + } + if p&0o2000 != 0 { + m |= os.ModeSetgid + } + if p&0o1000 != 0 { + m |= os.ModeSticky + } + switch p & 0o170000 { + case 0o040000: + m |= os.ModeDir + case 0o120000: + m |= os.ModeSymlink + case 0o020000: + m |= os.ModeDevice | os.ModeCharDevice + case 0o060000: + m |= os.ModeDevice + case 0o010000: + m |= os.ModeNamedPipe + case 0o140000: + m |= os.ModeSocket + case 0o100000: + // regular file: no extra bits + } + return m +} + +// Decode merges metadata from a HeadObject / GetObject response into defaults. +// Missing keys leave the corresponding field of `defaults` untouched, which is +// the behavior a POSIX filesystem usually wants: an object with no uid header +// inherits the mount's default uid, not zero. +func Decode(meta map[string]string, defaults Attrs) Attrs { + out := defaults + if s, ok := meta["uid"]; ok { + if v, err := strconv.ParseUint(s, 0, 32); err == nil { + out.UID = uint32(v) + } + } + if s, ok := meta["gid"]; ok { + if v, err := strconv.ParseUint(s, 0, 32); err == nil { + out.GID = uint32(v) + } + } + if s, ok := meta["mode"]; ok { + if v, err := strconv.ParseUint(s, 0, 32); err == nil { + out.Mode = GoFileMode(uint32(v)) + } + } + if s, ok := meta["rdev"]; ok { + if v, err := strconv.ParseUint(s, 0, 32); err == nil { + out.Rdev = uint32(v) + } + } + if s, ok := meta["mtime"]; ok { + if v, err := strconv.ParseInt(s, 0, 64); err == nil { + out.Mtime = time.Unix(v, 0) + } + } + if s, ok := meta["--symlink-target"]; ok { + if dec, err := url.QueryUnescape(s); err == nil { + out.SymlinkTarget = dec + } + } + return out +} +``` + +Used at the call site: + +```go +head, err := client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String("mybucket"), + Key: aws.String("path/to/file"), +}) +if err != nil { + return err +} + +attrs := unixmeta.Decode(head.Metadata, unixmeta.Attrs{ + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0o644, + // Mtime defaults to the object's S3 LastModified if mtime is missing: + Mtime: aws.ToTime(head.LastModified), +}) +``` + +## Parsing notes + +- `strconv.ParseUint(s, 0, ...)` with base `0` accepts decimal, `0o`-prefixed + octal, and `0x`-prefixed hex. Writers should emit decimal; readers should be + liberal. +- Treat malformed values as missing — parse errors fall through to the default + rather than failing the whole lookup. A single bad header should not make a + file unreadable. +- The `mode` header carries both the permission bits and the file-type bits. + When reapplying it to an in-memory inode, mask out the type bits if you only + want to update permissions (for `chmod`), and mask out the permission bits if + you only want to update the type (rare — typically set once at creation). + +## Directories and zero-byte objects + +Directories are zero-byte S3 objects whose key ends in `/`. They carry the same +metadata headers as regular files, with `mode` containing the directory type bit +(`0o040000`). Symlinks are likewise zero-byte objects; the link target lives +entirely in the `--symlink-target` header, not in the object body. diff --git a/internal/s3fs/basic.go b/internal/s3fs/basic.go index 6f3402c..d0c7ca0 100644 --- a/internal/s3fs/basic.go +++ b/internal/s3fs/basic.go @@ -13,7 +13,6 @@ import ( "strings" "time" - "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/smithy-go" "github.com/go-git/go-billy/v6" @@ -110,10 +109,10 @@ func (fs3 *S3FS) OpenFile(filename string, flag int, perm os.FileMode) (billy.Fi return nil, &os.PathError{Op: "open", Path: filename, Err: fs.ErrNotExist} case O_WRONLY: - return newS3WriteFile(fs3.client, fs3.bucket, key, filename) + return newS3WriteFile(fs3.client, fs3.bucket, key, filename, fs3.unixMeta) case O_WRMULTIPART: - return newS3MultipartUploadFile(fs3.client, fs3.bucket, key, filename) + return newS3MultipartUploadFile(fs3.client, fs3.bucket, key, filename, fs3.unixMeta) default: return nil, errors.New("unsupported open flag") @@ -140,11 +139,7 @@ func (fs3 *S3FS) Stat(filename string) (os.FileInfo, error) { Key: &key, }) if err == nil { - return newFileInfo( - path.Base(key), - aws.ToInt64(head.ContentLength), - aws.ToTime(head.LastModified), - ), nil + return newFileInfoFromHead(path.Base(key), head, fs3.unixMeta), nil } var apiErr smithy.APIError diff --git a/internal/s3fs/chroot.go b/internal/s3fs/chroot.go index c081c7b..5c7a601 100644 --- a/internal/s3fs/chroot.go +++ b/internal/s3fs/chroot.go @@ -23,6 +23,7 @@ func (fs3 *S3FS) Chroot(path string) (billy.Filesystem, error) { bucket: fs3.bucket, root: p, separator: fs3.separator, + unixMeta: fs3.unixMeta, temps: make(map[string]*tempBuffer), } return nfs, nil diff --git a/internal/s3fs/file.go b/internal/s3fs/file.go index e4a5685..54433c2 100644 --- a/internal/s3fs/file.go +++ b/internal/s3fs/file.go @@ -15,8 +15,24 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/tigrisdata/storage-go" "go.uber.org/atomic" + "tangled.org/xeiaso.net/objgit/internal/s3fs/unixmeta" ) +// newFileMetadata returns the x-amz-meta-* map to attach to a newly written +// object, or nil when the Unix-metadata feature is disabled. New files take the +// session's default owner and a mode of 0666 masked by the session umask. +func newFileMetadata(cfg *unixMetaConfig) map[string]string { + if cfg == nil { + return nil + } + return unixmeta.Encode(unixmeta.Attrs{ + UID: cfg.uid, + GID: cfg.gid, + Mode: 0o666 &^ cfg.umask, + Mtime: time.Now(), + }) +} + const ( ModeMultipartUpload os.FileMode = fs.ModePerm + 1 // Custom os.FileMode for S3 multipart upload ) @@ -159,23 +175,25 @@ func (f *s3ReadFile) Stat() (fs.FileInfo, error) { // Upon creation, a buffer is created to store the file contents. Upon close, // the file is uploaded to S3. type s3WriteFile struct { - client *storage.Client // s3 skd client - bucket string // S3 bucket name - key string // File object's key in S3 - name string // Root-relative path as presented to Open - closed bool // Is the file closed? - buf *bytes.Buffer // Buffer for storing the file before it's uploaded + client *storage.Client // s3 skd client + bucket string // S3 bucket name + key string // File object's key in S3 + name string // Root-relative path as presented to Open + closed bool // Is the file closed? + buf *bytes.Buffer // Buffer for storing the file before it's uploaded + unixMeta *unixMetaConfig // optional POSIX attribute defaults (nil = disabled) } // newS3WriteFile creates a new s3WriteFile. key is the full S3 object key; name // is the root-relative path the caller passed to Open (returned by Name). -func newS3WriteFile(client *storage.Client, bucket, key, name string) (*s3WriteFile, error) { +func newS3WriteFile(client *storage.Client, bucket, key, name string, cfg *unixMetaConfig) (*s3WriteFile, error) { return &s3WriteFile{ - client: client, - bucket: bucket, - key: key, - name: name, - buf: bytes.NewBuffer(nil), + client: client, + bucket: bucket, + key: key, + name: name, + buf: bytes.NewBuffer(nil), + unixMeta: cfg, }, nil } @@ -229,9 +247,10 @@ func (f *s3WriteFile) Close() error { // Run the GetObject operation // TODO: Currently `res` is not used. Should it be? _, err := f.client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &f.bucket, - Key: &f.key, - Body: body, + Bucket: &f.bucket, + Key: &f.key, + Body: body, + Metadata: newFileMetadata(f.unixMeta), }) if err != nil { return fmt.Errorf("unable to perform GetObject operation: %w", err) @@ -273,17 +292,19 @@ type s3MultipartUploadFile struct { // newS3MultipartUploadFile creates a new s3MultipartUploadFile. key is the full // S3 object key; name is the root-relative path passed to Open. -func newS3MultipartUploadFile(client *storage.Client, bucket, key, name string) (*s3MultipartUploadFile, error) { +func newS3MultipartUploadFile(client *storage.Client, bucket, key, name string, cfg *unixMetaConfig) (*s3MultipartUploadFile, error) { // TODO: Check if the file exists // ... // Create the context ctx := context.TODO() // TODO: How can user-supplied contexts be supported? - // Run the GetObject operation + // Run the GetObject operation. POSIX attributes (if enabled) must be set + // now: CompleteMultipartUpload cannot attach user metadata. res, err := client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &key, + Bucket: &bucket, + Key: &key, + Metadata: newFileMetadata(cfg), }) if err != nil { return nil, fmt.Errorf("unable to create multipart upload: %w", err) diff --git a/internal/s3fs/fileinfo.go b/internal/s3fs/fileinfo.go index 7fb243a..9171972 100644 --- a/internal/s3fs/fileinfo.go +++ b/internal/s3fs/fileinfo.go @@ -5,15 +5,25 @@ import ( "os" "time" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" + "tangled.org/xeiaso.net/objgit/internal/s3fs/unixmeta" ) +// FileStat is the value returned by simpleFileInfo.Sys() when the +// Unix-metadata feature is enabled. It carries the raw numeric owner and group +// so consumers can resolve them to names however they like. +type FileStat struct { + UID, GID uint32 +} + // simpleFileInfo implements os.FileInfo type simpleFileInfo struct { name string size int64 mode os.FileMode modTime time.Time + sys *FileStat } func newFileInfo(name string, size int64, modTime time.Time) os.FileInfo { @@ -33,11 +43,43 @@ func newDirInfo(name string) os.FileInfo { } } -func (fi simpleFileInfo) Name() string { return fi.name } -func (fi simpleFileInfo) Size() int64 { return fi.size } -func (fi simpleFileInfo) Mode() os.FileMode { return fi.mode } -func (fi simpleFileInfo) IsDir() bool { return fi.mode.IsDir() } -func (fi simpleFileInfo) Sys() interface{} { return nil } +// newFileInfoFromHead builds a FileInfo from a HeadObject response. When cfg is +// nil the Unix-metadata feature is off and the result matches newFileInfo; +// otherwise the x-amz-meta-* attributes are decoded, falling back to the +// session defaults for any header that is missing. +func newFileInfoFromHead(name string, head *s3.HeadObjectOutput, cfg *unixMetaConfig) os.FileInfo { + size := aws.ToInt64(head.ContentLength) + modTime := aws.ToTime(head.LastModified) + if cfg == nil { + return newFileInfo(name, size, modTime) + } + + attrs := unixmeta.Decode(head.Metadata, unixmeta.Attrs{ + UID: cfg.uid, + GID: cfg.gid, + Mode: 0o666 &^ cfg.umask, + Mtime: modTime, + }) + + return simpleFileInfo{ + name: name, + size: size, + mode: attrs.Mode, + modTime: attrs.Mtime, + sys: &FileStat{UID: attrs.UID, GID: attrs.GID}, + } +} + +func (fi simpleFileInfo) Name() string { return fi.name } +func (fi simpleFileInfo) Size() int64 { return fi.size } +func (fi simpleFileInfo) Mode() os.FileMode { return fi.mode } +func (fi simpleFileInfo) IsDir() bool { return fi.mode.IsDir() } +func (fi simpleFileInfo) Sys() any { + if fi.sys == nil { + return nil + } + return fi.sys +} func (fi simpleFileInfo) ModTime() time.Time { return fi.modTime } type enrichedFileInfo struct { diff --git a/internal/s3fs/filesystem.go b/internal/s3fs/filesystem.go index 1db7b48..74d323c 100644 --- a/internal/s3fs/filesystem.go +++ b/internal/s3fs/filesystem.go @@ -2,6 +2,7 @@ package s3fs import ( "fmt" + "os" "path" "strings" "sync" @@ -14,11 +15,20 @@ const ( DefaultSeparator = "/" ) +// unixMetaConfig holds the session defaults used when the optional Unix-metadata +// feature is enabled. A nil *unixMetaConfig means the feature is off and the +// filesystem behaves as if no POSIX attributes exist. +type unixMetaConfig struct { + uid, gid uint32 + umask os.FileMode +} + type S3FS struct { client *storage.Client bucket string root string separator string + unixMeta *unixMetaConfig // temps holds TempFile-backed buffers keyed by canonical S3 key, so a // subsequent Open of the same path returns a reader over the same bytes @@ -27,19 +37,37 @@ type S3FS struct { temps map[string]*tempBuffer } +// Option configures an S3FS at construction time. +type Option func(*S3FS) + +// WithUnixMetadata enables storing and reading POSIX file attributes as S3 user +// metadata (see the unixmeta package). uid and gid are the numeric owner/group +// recorded on newly written objects; callers resolve any names to numbers +// themselves. umask is applied to the default mode (0666 for files) when +// writing. Without this option the filesystem stores no attributes. +func WithUnixMetadata(uid, gid uint32, umask os.FileMode) Option { + return func(fs3 *S3FS) { + fs3.unixMeta = &unixMetaConfig{uid: uid, gid: gid, umask: umask} + } +} + // NewS3FS creates a new S3FS Filesystem. -func NewS3FS(client *storage.Client, bucket string) (billy.Filesystem, error) { +func NewS3FS(client *storage.Client, bucket string, opts ...Option) (billy.Filesystem, error) { // Check for a non-nil client if client == nil { return nil, fmt.Errorf("s3 client cannot be nil") } - return &S3FS{ + fs3 := &S3FS{ client: client, bucket: bucket, root: "", separator: DefaultSeparator, temps: make(map[string]*tempBuffer), - }, nil + } + for _, opt := range opts { + opt(fs3) + } + return fs3, nil } // Capabilities returns the filesystem capabilities. diff --git a/internal/s3fs/unixmeta/unixmeta.go b/internal/s3fs/unixmeta/unixmeta.go new file mode 100644 index 0000000..9ce2816 --- /dev/null +++ b/internal/s3fs/unixmeta/unixmeta.go @@ -0,0 +1,207 @@ +// Package unixmeta encodes and decodes POSIX file attributes (owner, group, +// permissions, timestamps) as S3 user-defined metadata (x-amz-meta-* headers). +// +// S3 does not natively model Unix attributes, so a POSIX filesystem layered on +// top of object storage keeps them in a small set of string-valued metadata +// headers. The on-the-wire format is documented in +// docs/reference/how-tigris-fs-unix-metadata.md. +package unixmeta + +import ( + "net/url" + "os" + "os/user" + "strconv" + "time" +) + +// Metadata keys (without the x-amz-meta- prefix the S3 SDK prepends). +const ( + keyUID = "uid" + keyGID = "gid" + keyMode = "mode" + keyRdev = "rdev" + keyMtime = "mtime" + keySymlinkTarget = "--symlink-target" +) + +// POSIX file-type mask and the bits stored in the high part of mode_t. +const ( + modeTypeMask = 0o170000 + modeRegular = 0o100000 + modeDir = 0o040000 + modeSymlink = 0o120000 + modeBlock = 0o060000 + modeChar = 0o020000 + modeFIFO = 0o010000 + modeSocket = 0o140000 +) + +// Attrs is the set of POSIX attributes recorded on an object. +type Attrs struct { + UID, GID uint32 + Mode os.FileMode + Rdev uint32 + Mtime time.Time + SymlinkTarget string // "" if not a symlink +} + +// PosixMode converts a Go os.FileMode to the POSIX mode_t integer stored in +// x-amz-meta-mode. +func PosixMode(m os.FileMode) uint32 { + out := uint32(m.Perm()) // low 9 permission bits + if m&os.ModeSetuid != 0 { + out |= 0o4000 + } + if m&os.ModeSetgid != 0 { + out |= 0o2000 + } + if m&os.ModeSticky != 0 { + out |= 0o1000 + } + switch { + case m&os.ModeDir != 0: + out |= modeDir + case m&os.ModeSymlink != 0: + out |= modeSymlink + case m&os.ModeDevice != 0 && m&os.ModeCharDevice != 0: + out |= modeChar + case m&os.ModeDevice != 0: + out |= modeBlock + case m&os.ModeNamedPipe != 0: + out |= modeFIFO + case m&os.ModeSocket != 0: + out |= modeSocket + default: + out |= modeRegular + } + return out +} + +// GoFileMode is the inverse of PosixMode. +func GoFileMode(p uint32) os.FileMode { + m := os.FileMode(p & 0o777) + if p&0o4000 != 0 { + m |= os.ModeSetuid + } + if p&0o2000 != 0 { + m |= os.ModeSetgid + } + if p&0o1000 != 0 { + m |= os.ModeSticky + } + switch p & modeTypeMask { + case modeDir: + m |= os.ModeDir + case modeSymlink: + m |= os.ModeSymlink + case modeChar: + m |= os.ModeDevice | os.ModeCharDevice + case modeBlock: + m |= os.ModeDevice + case modeFIFO: + m |= os.ModeNamedPipe + case modeSocket: + m |= os.ModeSocket + case modeRegular: + // regular file: no extra bits + } + return m +} + +// Encode produces the user-metadata map for a PutObject call. Pass the result +// as PutObjectInput.Metadata; the S3 SDK prepends x-amz-meta- and lowercases +// the keys. +func Encode(a Attrs) map[string]string { + mode := PosixMode(a.Mode) + m := map[string]string{ + keyUID: strconv.FormatUint(uint64(a.UID), 10), + keyGID: strconv.FormatUint(uint64(a.GID), 10), + keyMode: strconv.FormatUint(uint64(mode), 10), + keyMtime: strconv.FormatInt(a.Mtime.Unix(), 10), + } + if a.Mode&(os.ModeDevice|os.ModeCharDevice) != 0 { + m[keyRdev] = strconv.FormatUint(uint64(a.Rdev), 10) + } + if a.SymlinkTarget != "" { + m[keySymlinkTarget] = url.QueryEscape(a.SymlinkTarget) + } + return m +} + +// Decode merges metadata from a HeadObject / GetObject response into defaults. +// Missing keys leave the corresponding field of defaults untouched, which is +// the behavior a POSIX filesystem usually wants: an object with no uid header +// inherits the mount's default uid, not zero. Malformed values are treated as +// missing so a single bad header doesn't make a file unreadable. +func Decode(meta map[string]string, defaults Attrs) Attrs { + out := defaults + if s, ok := meta[keyUID]; ok { + if v, err := strconv.ParseUint(s, 0, 32); err == nil { + out.UID = uint32(v) + } + } + if s, ok := meta[keyGID]; ok { + if v, err := strconv.ParseUint(s, 0, 32); err == nil { + out.GID = uint32(v) + } + } + if s, ok := meta[keyMode]; ok { + if v, err := strconv.ParseUint(s, 0, 32); err == nil { + out.Mode = GoFileMode(uint32(v)) + } + } + if s, ok := meta[keyRdev]; ok { + if v, err := strconv.ParseUint(s, 0, 32); err == nil { + out.Rdev = uint32(v) + } + } + if s, ok := meta[keyMtime]; ok { + if v, err := strconv.ParseInt(s, 0, 64); err == nil { + out.Mtime = time.Unix(v, 0) + } + } + if s, ok := meta[keySymlinkTarget]; ok { + if dec, err := url.QueryUnescape(s); err == nil { + out.SymlinkTarget = dec + } + } + return out +} + +// LookupUID resolves a user name to a numeric UID. It first consults the host +// passwd database via os/user; if the name is not found there it is parsed as a +// decimal UID. This lets callers pass either "alice" or "1000" and works in +// containers that lack a matching passwd entry. The package never calls this +// itself — callers decide whether to resolve names. +func LookupUID(name string) (uint32, error) { + if u, err := user.Lookup(name); err == nil { + v, perr := strconv.ParseUint(u.Uid, 10, 32) + if perr != nil { + return 0, perr + } + return uint32(v), nil + } + v, err := strconv.ParseUint(name, 10, 32) + if err != nil { + return 0, err + } + return uint32(v), nil +} + +// LookupGID resolves a group name to a numeric GID, with the same name-or-number +// semantics as LookupUID. +func LookupGID(name string) (uint32, error) { + if g, err := user.LookupGroup(name); err == nil { + v, perr := strconv.ParseUint(g.Gid, 10, 32) + if perr != nil { + return 0, perr + } + return uint32(v), nil + } + v, err := strconv.ParseUint(name, 10, 32) + if err != nil { + return 0, err + } + return uint32(v), nil +} diff --git a/internal/s3fs/unixmeta/unixmeta_test.go b/internal/s3fs/unixmeta/unixmeta_test.go new file mode 100644 index 0000000..e89f38c --- /dev/null +++ b/internal/s3fs/unixmeta/unixmeta_test.go @@ -0,0 +1,132 @@ +package unixmeta + +import ( + "os" + "testing" + "time" +) + +func TestModeRoundTrip(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + name string + mode os.FileMode + }{ + {name: "regular file rw-r--r--", mode: 0o644}, + {name: "regular file rwxr-xr-x", mode: 0o755}, + {name: "directory", mode: os.ModeDir | 0o755}, + {name: "symlink", mode: os.ModeSymlink | 0o777}, + {name: "block device", mode: os.ModeDevice | 0o660}, + {name: "char device", mode: os.ModeDevice | os.ModeCharDevice | 0o620}, + {name: "fifo", mode: os.ModeNamedPipe | 0o644}, + {name: "socket", mode: os.ModeSocket | 0o755}, + {name: "setuid", mode: os.ModeSetuid | 0o755}, + {name: "setgid", mode: os.ModeSetgid | 0o755}, + {name: "sticky dir", mode: os.ModeDir | os.ModeSticky | 0o777}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := GoFileMode(PosixMode(tt.mode)) + if got != tt.mode { + t.Logf("want: %v (%#o)", tt.mode, uint32(tt.mode)) + t.Logf("got: %v (%#o)", got, uint32(got)) + t.Error("mode did not survive round trip") + } + }) + } +} + +func TestPosixModeKnownValue(t *testing.T) { + t.Parallel() + + // 0o100644 == 33188: regular file with rw-r--r--, per the reference doc. + if got := PosixMode(0o644); got != 0o100644 { + t.Errorf("PosixMode(0o644) = %#o, want %#o", got, 0o100644) + } +} + +func TestEncodeDecodeRoundTrip(t *testing.T) { + t.Parallel() + + mtime := time.Unix(1716700000, 0) + + for _, tt := range []struct { + name string + in Attrs + }{ + { + name: "regular file", + in: Attrs{UID: 1000, GID: 1000, Mode: 0o644, Mtime: mtime}, + }, + { + name: "directory", + in: Attrs{UID: 0, GID: 0, Mode: os.ModeDir | 0o755, Mtime: mtime}, + }, + { + name: "char device with rdev", + in: Attrs{UID: 0, GID: 5, Mode: os.ModeDevice | os.ModeCharDevice | 0o620, Rdev: 1280, Mtime: mtime}, + }, + { + name: "symlink with awkward target", + in: Attrs{UID: 1000, GID: 1000, Mode: os.ModeSymlink | 0o777, Mtime: mtime, SymlinkTarget: "/etc/has spaces/and%percent"}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := Decode(Encode(tt.in), Attrs{}) + if got.UID != tt.in.UID || got.GID != tt.in.GID { + t.Errorf("uid/gid: want %d/%d, got %d/%d", tt.in.UID, tt.in.GID, got.UID, got.GID) + } + if got.Mode != tt.in.Mode { + t.Errorf("mode: want %v, got %v", tt.in.Mode, got.Mode) + } + if !got.Mtime.Equal(tt.in.Mtime) { + t.Errorf("mtime: want %v, got %v", tt.in.Mtime, got.Mtime) + } + if got.SymlinkTarget != tt.in.SymlinkTarget { + t.Errorf("symlink target: want %q, got %q", tt.in.SymlinkTarget, got.SymlinkTarget) + } + if tt.in.Rdev != 0 && got.Rdev != tt.in.Rdev { + t.Errorf("rdev: want %d, got %d", tt.in.Rdev, got.Rdev) + } + }) + } +} + +func TestDecodeMissingKeysKeepDefaults(t *testing.T) { + t.Parallel() + + defaults := Attrs{UID: 501, GID: 20, Mode: 0o644, Mtime: time.Unix(42, 0)} + got := Decode(map[string]string{}, defaults) + if got != defaults { + t.Logf("want: %+v", defaults) + t.Logf("got: %+v", got) + t.Error("empty metadata should leave defaults untouched") + } +} + +func TestDecodeMalformedTreatedAsMissing(t *testing.T) { + t.Parallel() + + defaults := Attrs{UID: 501, GID: 20, Mode: 0o644, Mtime: time.Unix(42, 0)} + meta := map[string]string{ + "uid": "not-a-number", + "gid": "99", + "mode": "garbage", + "mtime": "also-bad", + } + got := Decode(meta, defaults) + if got.UID != defaults.UID { + t.Errorf("malformed uid should fall back to default: want %d, got %d", defaults.UID, got.UID) + } + if got.GID != 99 { + t.Errorf("valid gid should be parsed: want 99, got %d", got.GID) + } + if got.Mode != defaults.Mode { + t.Errorf("malformed mode should fall back to default: want %v, got %v", defaults.Mode, got.Mode) + } + if !got.Mtime.Equal(defaults.Mtime) { + t.Errorf("malformed mtime should fall back to default: want %v, got %v", defaults.Mtime, got.Mtime) + } +}