diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 26bddb978..07f95ecb0 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -35,7 +35,8 @@ jobs: - name: Check gofmt run: | mapfile -t unformatted < <(gofmt -l main.go cmd internal \ - buildscripts/rebrand-guard buildscripts/helm-migration-guard) + buildscripts/rebrand-guard buildscripts/helm-migration-guard \ + buildscripts/release-dependency-guard) if [ "${#unformatted[@]}" -ne 0 ]; then echo "The following files are not gofmt-formatted:" printf '%s\n' "${unformatted[@]}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bbc2d1067..635215dcc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,6 +20,7 @@ permissions: id-token: write attestations: write artifact-metadata: write + actions: read concurrency: group: release-${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }} @@ -109,14 +110,29 @@ jobs: { echo "RELEASE_TAG=${TAG}" + echo "RELEASE_COMMIT=${HEAD_COMMIT}" echo "PKG_VERSION=${PKG_VERSION}" echo "LDFLAGS=${LDFLAGS}" } >> "${GITHUB_ENV}" echo "Release tag: ${TAG}" + echo "Release commit: ${HEAD_COMMIT}" echo "Package version: ${PKG_VERSION}" echo "LDFLAGS: ${LDFLAGS}" + - name: Check release commit is the tip of main with green CI + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + buildscripts/check-release-commit.sh "${RELEASE_COMMIT}" + + - name: Verify coordinated release dependencies + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GOWORK: "off" + run: go run ./buildscripts/release-dependency-guard + - name: Check existing release state env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -140,7 +156,7 @@ jobs: - name: Build Draft release with GoReleaser uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3 with: - version: v2.18.3 + version: v2.20.1 args: release --clean --skip=validate --config .github/goreleaser.yml env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test-release.yml b/.github/workflows/test-release.yml index 1b3c702a7..7d24eba53 100644 --- a/.github/workflows/test-release.yml +++ b/.github/workflows/test-release.yml @@ -25,6 +25,9 @@ on: - "buildscripts/minio-upgrade.sh" - "buildscripts/sign-release-rpms.sh" - "buildscripts/verify-build-provenance.sh" + - "buildscripts/check-release-commit.sh" + - "buildscripts/check-release-commit_test.sh" + - "buildscripts/release-dependency-guard/**" - "buildscripts/check-release-state.sh" - "buildscripts/check-release-state_test.sh" - "buildscripts/verify-rebrand.sh" @@ -97,7 +100,7 @@ jobs: - name: GoReleaser config check uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3 with: - version: v2.18.3 + version: v2.20.1 args: check --config .github/goreleaser.yml - name: Validate Helm chart and legacy upgrade identity @@ -111,7 +114,7 @@ jobs: - name: Build snapshot artifacts uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3 with: - version: v2.18.3 + version: v2.20.1 # A pull-request snapshot has no trusted release identity. Exercise # the SBOM/checksum pipeline here, and reserve keyless signing for # the tag-triggered release workflow with GitHub OIDC. @@ -537,6 +540,8 @@ jobs: bash -n buildscripts/minio-upgrade.sh bash -n buildscripts/sign-release-rpms.sh bash -n buildscripts/verify-build-provenance.sh + bash -n buildscripts/check-release-commit.sh + bash -n buildscripts/check-release-commit_test.sh bash -n buildscripts/check-release-state.sh bash -n buildscripts/check-release-state_test.sh bash -n buildscripts/verify-rebrand.sh @@ -553,6 +558,8 @@ jobs: test -x buildscripts/package-release.sh test -x buildscripts/sign-release-rpms.sh test -x buildscripts/verify-build-provenance.sh + test -x buildscripts/check-release-commit.sh + test -x buildscripts/check-release-commit_test.sh test -x buildscripts/check-release-state.sh test -x buildscripts/check-release-state_test.sh test -x buildscripts/verify-rebrand.sh @@ -561,4 +568,6 @@ jobs: test -x buildscripts/package/preremove.sh test -x buildscripts/package/lifecycle_test.sh test -x dockerscripts/docker-entrypoint_test.sh + buildscripts/check-release-commit_test.sh + GOWORK=off go test ./buildscripts/release-dependency-guard buildscripts/check-release-state_test.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cca54290..fca5465df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,37 +1,74 @@ # Changelog -## Unreleased +## RELEASE.2026-11-12T00-00-00Z -These changes follow Server 20261109 and have no assigned release version yet. -Console and mcli policy-editing repairs require matching component releases; -the current Server dependency pins still select Console v2.7.0 and mcli 20261109. -See the [server compatibility notes](https://silo.pgsty.com/compatibility/server/) -for the candidate behavior and upgrade boundaries. +Package version: `20261112000000.0.0`. Publication status and artifacts are +listed in [GitHub Releases](https://github.com/pgsty/silo/releases). This +section covers the changes after the published 20261109 source `d998292387cc`, +adopting Go `1.27.4`, `silo-pkg v3.16.1`, embedded Console `v2.7.1`, and +bundled `mcli 20261112`. See the +[server compatibility notes](https://silo.pgsty.com/compatibility/server/) +for behavior and upgrade boundaries. +- Release checks verify the selected pkg and embedded Console versions and + match the bundled mcli release to its source module, independently of local + workspaces, before building release artifacts. Deploy SILO and mcli + `RELEASE.2026-11-12T00-00-00Z` by default from Helm chart `7.0.5`. - Replica writes and copies use the destination version's retention, legal - hold and tags under its mutation lock. Cross-pool reconciliation preserves - COMPLIANCE protection regardless of pool order, while authorized changes to - GOVERNANCE retention and legal hold remain possible. Local lock revisions - advance past stored revisions, and forced prefix deletion refuses an - unavailable Object Lock configuration ([#336](https://github.com/pgsty/silo/issues/336)). + hold and tags under its mutation lock, verify staged replica content and + SSE-C/compressed metadata against stored locked versions before committing, + and prevent newer peer timestamps from weakening unexpired COMPLIANCE + retention. Cross-pool reconciliation preserves COMPLIANCE protection + regardless of pool order, while authorized changes to GOVERNANCE retention + and legal hold remain possible. Local lock revisions advance past stored + revisions, unaddressed deletes under versioning suspension or excluded + prefixes enforce Object Lock across single-pool and multi-pool deployments + and fail closed when versioning or Object Lock metadata cannot be read, + bucket metadata imports require versioning across prefix exclusions as well + as suspension, and peer replication-trust headers must be signed with + header-based SigV4 ([#336](https://github.com/pgsty/silo/issues/336)). - Tagging proxies (`GET`, `PUT` and `DELETE`) authenticate and authorize every present replica, including its stored tags, fail closed on unreadable targets, and skip missing replicas caused by replication lag unless all targets are - absent. Authenticated unsigned-trailer uploads enforce the request time - window; anonymous uploads allowed by bucket policy remain supported + absent; active-active `GET` and `HEAD` proxies enforce `s3:ExistingObjectTag` + and `s3:versionid` before returning peer responses and forward + `X-Amz-Tagging-Directive: ACCESS` on `HEAD`. Authenticated unsigned-trailer + uploads enforce the request time window and header signatures, SigV4 header, + presigned, streaming and POST policy verifiers bind the credential-scope date + to the signed request timestamp, streaming trailer readers bound trailer memory + and surface non-EOF stream errors without hanging on negative + `X-Amz-Decoded-Content-Length`, and bucket metadata cache loads no longer + overwrite fresher concurrent updates ([#337](https://github.com/pgsty/silo/issues/337), [#338](https://github.com/pgsty/silo/issues/338)). -- `GetObjectAttributes` evaluates both Attributes and GetObject permissions - against the addressed version's stored tags, using version actions for an - explicit version. ZIP GET, HEAD and member listings also authorize the ZIP - object's stored tags; listing members requires both `s3:ListBucket` and - `s3:GetObject`. These reads no longer write an archive index back to the - object. Missing-object masking still uses `s3:ListBucket`, and authorized - storage failures retain their server error ([#342](https://github.com/pgsty/silo/issues/342)). +- Object tag and version authorization now applies consistently across read, + copy, list and archive endpoints: `GetObject`, `HeadObject` and + `SelectObjectContent` evaluate `s3:ExistingObjectTag` and `s3:versionid` + (preserving retryable `5xx` server errors for callers authorized by tag + conditions when storage is unreadable, resolving the 20261109 limitation in + [PR #294](https://github.com/pgsty/silo/pull/294)); `CopyObject` and + `UploadPartCopy` authorize source `s3:ExistingObjectTag` and `s3:versionid` + as well as destination `s3:RequestObjectTag` on copied tags; `X-Amz-Tagging` + is returned on `GET`/`HEAD` (`X-Amz-Tagging-Directive: ACCESS`) only when + `s3:GetObjectTagging` / `s3:GetObjectVersionTagging` (or `s3:ReplicateTags`) + permits; `ListObjectsV2` / `ListObjectVersions` with `metadata=true` evaluate + current-object and per-version metadata permissions respectively, using stored tags; and `GetObjectAttributes`, + tagging handlers (`GET`/`PUT`/`DELETE`) and ZIP `GET`/`HEAD`/member listings + authorize stored tags and version actions without mutating unindexed archives + ([#342](https://github.com/pgsty/silo/issues/342)). +- Policy parsing rejects unknown JSON fields, trailing tokens and malformed + principals across admin IAM policy/user/service-account endpoints, STS + session policies, IAM imports/replication, bucket policies and bucket metadata + imports instead of silently accepting empty or wildcard fallbacks. Validate + the complete imported policy file before applying entries; retain supported + historical snapshot semantics and explicit null bucket-policy replication + tombstones. New session policies retain strict resource validation. - Every explicit version removal in S3 `DeleteObjects`, including in buckets without Object Lock, now uses the sequential single-object path to check metadata, retention and replication under the mutation lock. Large version - purges can take longer; entries without a version ID retain the bulk path. + purges can take longer. Unaddressed deletes that can replace a locked null + version also use the single-object path; other unaddressed deletes retain + bulk execution. - Decommission rescans keys hidden below another key only while source versions are actually removed, resolving the 20261109 limitation recorded in [#293](https://github.com/pgsty/silo/issues/293). Decommission and rebalance @@ -52,52 +89,47 @@ for the candidate behavior and upgrade boundaries. disabled service accounts and groups, reports group failures, and excludes derived session-policy claims from exported and replicated credentials ([#345](https://github.com/pgsty/silo/issues/345)). -- Historical-version self-copy plans its destination pool and data rewrite +- Lifecycle history revision counting treats old and healed `xl.meta` headers + for the same version as one revision ([#332](https://github.com/pgsty/silo/issues/332)), + and restore expiry no longer turns a time-derived data-directory flag into a + different version revision; mixed healed headers can reach quorum and expired + local restore data can be cleaned up ([#344](https://github.com/pgsty/silo/issues/344)). + Historical-version self-copy plans its destination pool and data rewrite before encryption or compression, preserving multi-block compression seek indexes so a source version in another pool creates a readable new version. In-place metadata updates locate the exact requested version and preserve authoritative content on moving pools, while copies that require encryption or checksum recomputation select an active pool - ([#343](https://github.com/pgsty/silo/issues/343)). Restore expiry no longer - turns a time-derived data-directory flag into a different version revision; - mixed healed headers can reach quorum and expired local restore data can be - cleaned up ([#344](https://github.com/pgsty/silo/issues/344)). -- Batch KMS rotation decodes the configured encryption context on execution - and after reload or retry, rejecting invalid UTF-8 before changing an object - key and without adding validation-probe fields to object metadata + ([#343](https://github.com/pgsty/silo/issues/343)). +- Batch KMS rotation decodes and validates the configured encryption context + (and incoming HTTP SSE-KMS context headers) as strict UTF-8 JSON before + changing an object key and without adding validation-probe fields to object + metadata, supports `SSE-S3` to `SSE-KMS` key migration and multipart metadata + preservation, and rewrites legacy `XLV1` objects onto `XLv2` streams instead + of failing in-place metadata updates ([#346](https://github.com/pgsty/silo/issues/346)). Batch variable preflight follows the pinned policy evaluator's literal and substitution rules, including BinaryEquals, while retaining refusal of unavailable request context. This does not add AWS special-variable escape semantics ([#347](https://github.com/pgsty/silo/issues/347)). -- Service-account create and edit normalize a missing policy Version to the - default. Edit distinguishes an omitted or null policy (preserve) from an - explicit empty document (return to inheritance). Matching Console changes - preserve inherited policy on metadata edits; matching mcli changes accept - explicit empty policies only on edit, keeping create and named-policy - validation strict ([#339](https://github.com/pgsty/silo/issues/339), +- Service-account and access-key create/edit normalize a missing policy Version + to `2012-10-17`, reject malformed or unknown policy fields, and distinguish an + omitted or `null` policy (preserve existing inline policy) from an explicit + empty policy document such as `{}` or `{"Statement":[]}` (reset to parent + inheritance), while keeping historical deny-all accounts carrying an empty + embedded policy restricted until explicitly reset + ([#339](https://github.com/pgsty/silo/issues/339), [Console #89](https://github.com/pgsty/silo-console/issues/89), - [mcli #78](https://github.com/pgsty/mc/issues/78)). - -- An edit of a service account or access key that names no policy keeps its - inline policy. `mcli admin user svcacct disable`, `enable` and `edit` without - `--policy`, the same `idp ldap accesskey` commands, and any other - `update-service-account` request without `newPolicy` removed the inline - policy at the site that took the request and, for an account of another user - than root, at its replication peers, and the account then had all its parent - is allowed. The inline policy is removed - only on request, by an empty policy document such as `{"Statement":[]}` - (`mcli admin user svcacct edit --policy`); a client that left the policy out - to remove it must send that document. A JSON `null` policy counts as none. - In the matching Console, turn off **Restrict beyond user policy** or enter - `{}` to remove an inline policy. Clearing the editor while restriction is - enabled prevents saving. The fix does not bring back a policy - that was removed, and a site of a replicated deployment that does not run the - fix still removes the policy on such an edit that it takes itself or receives - from another site that does not run it, so sites can differ. Upgrade every - site, then review on each site the accounts that `svcacct info` reports with - `Policy: implied`, and set the policy again on those that were meant to be - restricted. + [mcli #78](https://github.com/pgsty/mc/issues/78)). Previously, + `mcli admin user svcacct disable`, `enable` and `edit` without `--policy`, + the matching `admin accesskey` and `idp ldap/openid accesskey` commands, and + any other `update-service-account` request without `newPolicy` removed the + inline policy on the receiving site and on site-replication peers for + non-root parent users. In the matching Console, turn off **Restrict beyond + user policy** or enter `{}` to remove an inline policy; clearing the editor + while restriction is enabled prevents saving. Upgrade every site, then review + accounts that `svcacct info` reports with `Policy: implied` and re-apply + restrictions where needed. - S3 `DeleteObjects` on a single pool stops at an entry whose name fails the object name check: the entry returns `XMinioInvalidObjectName` and is not @@ -300,7 +332,7 @@ Known issues: `ListMultipartUploads` prefix and paging ([#217](https://github.com/pgsty/silo/issues/217)), replayed bucket creation under site replication ([PR #241](https://github.com/pgsty/silo/pull/241)), and `403` instead of the server error for reads allowed only by object tags while -the object is unreadable ([#293](https://github.com/pgsty/silo/issues/293)). +the object is unreadable ([PR #294](https://github.com/pgsty/silo/pull/294)). ## RELEASE.2026-10-26T00-00-00Z diff --git a/Dockerfile.distroless b/Dockerfile.distroless index b88c27cd2..40309a06a 100644 --- a/Dockerfile.distroless +++ b/Dockerfile.distroless @@ -14,7 +14,7 @@ RUN mkdir -p /prep/data && chmod 0777 /prep/data # Debian's bundle in distroless still trusts the roots the classic image # blocklists. Build the same bundle as Dockerfile.goreleaser so both variants # trust the same roots; it is architecture-neutral, so build it natively. -FROM --platform=$BUILDPLATFORM registry.access.redhat.com/ubi9/ubi:latest@sha256:16e9e43d99e181463513cc84a514859b1abb9460f1c4d9e208da4b4e51cd5cc3 AS certs +FROM --platform=$BUILDPLATFORM registry.access.redhat.com/ubi9/ubi:latest@sha256:8b212f75a0e5a156ac28b1c5c030d2e8a07715254e19d522286118279399af81 AS certs # Native root blocklist mitigation: https://access.redhat.com/security/vulnerabilities/RHSB-2026-012 RUN dnf -y install ca-certificates && \ if [ -d /etc/pki/ca-trust/source/blocklist ]; then \ diff --git a/Dockerfile.goreleaser b/Dockerfile.goreleaser index 4b8ed57dd..9f143c0c1 100644 --- a/Dockerfile.goreleaser +++ b/Dockerfile.goreleaser @@ -1,4 +1,4 @@ -FROM golang:1.27.3-alpine@sha256:d530cf60723abf1ffae0cd16bb546b379ec0550d2f3815e5b16ed69dbc00fba4 AS curl-build +FROM golang:1.27.4-alpine@sha256:0a5f7fe796599ff487605987432e7619ae45beeb2fba4c2ce0b907879932aebf AS curl-build ARG TARGETARCH COPY dockerscripts/build-static-curl.sh /build/build-static-curl RUN /bin/sh /build/build-static-curl @@ -9,7 +9,7 @@ COPY --from=curl-build /go/bin/curl /curl COPY --from=curl-build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt ENTRYPOINT ["/curl"] -FROM golang:1.27.3-alpine@sha256:d530cf60723abf1ffae0cd16bb546b379ec0550d2f3815e5b16ed69dbc00fba4 AS build +FROM golang:1.27.4-alpine@sha256:0a5f7fe796599ff487605987432e7619ae45beeb2fba4c2ce0b907879932aebf AS build ARG TARGETARCH @@ -17,9 +17,9 @@ ENV GOPATH=/go ENV CGO_ENABLED=0 ARG MC_REPO=pgsty/mc -ARG MC_VERSION=RELEASE.2026-11-09T00-00-00Z -ARG MC_AMD64_SHA256=79e6cf4ad00a9e5eb2e7ceccc6e6496e4af4ec096e59695515ea8f223c287e4a -ARG MC_ARM64_SHA256=5473c4b39a31dd66478f5c44686a09f3e9e37861a30f1faf18564fbe66118387 +ARG MC_VERSION=RELEASE.2026-11-12T00-00-00Z +ARG MC_AMD64_SHA256=68875c7fdd06a76af12b74753506df65226d1bec65a405d2617d76448c229ba2 +ARG MC_ARM64_SHA256=4ea59d2f278c96b25e380289fd2ee9bc49faa342687074f031805cb7b34fbe1a RUN apk add -U --no-cache \ ca-certificates \ @@ -69,7 +69,7 @@ RUN apk add -U --no-cache \ mkdir -p /go/share/mcli && \ cp /tmp/mcli-extract/LICENSE /tmp/mcli-extract/NOTICE /tmp/mcli-extract/CREDITS /go/share/mcli/ -FROM registry.access.redhat.com/ubi9/ubi:latest@sha256:16e9e43d99e181463513cc84a514859b1abb9460f1c4d9e208da4b4e51cd5cc3 AS certs +FROM registry.access.redhat.com/ubi9/ubi:latest@sha256:8b212f75a0e5a156ac28b1c5c030d2e8a07715254e19d522286118279399af81 AS certs # Native root blocklist mitigation: https://access.redhat.com/security/vulnerabilities/RHSB-2026-012 RUN dnf -y install ca-certificates && \ if [ -d /etc/pki/ca-trust/source/blocklist ]; then \ @@ -88,7 +88,7 @@ RUN dnf -y install ca-certificates && \ dnf clean all && \ rm -rf /var/cache/dnf -FROM registry.access.redhat.com/ubi9/ubi-micro:latest@sha256:5be877a2a280ae36eb9ab80ca4efb28bbf77c57abbba38137cdc37a03b8465e6 +FROM registry.access.redhat.com/ubi9/ubi-micro:latest@sha256:b59ac4f291d86027c2681f7a7b1cbf7448972fc55a6cb8e25cab5aa5eef5ce5f LABEL org.opencontainers.image.title="Silo" \ org.opencontainers.image.description="S3-Interface Libre Object Storage" \ diff --git a/buildscripts/check-release-commit.sh b/buildscripts/check-release-commit.sh new file mode 100755 index 000000000..e7f1aadb2 --- /dev/null +++ b/buildscripts/check-release-commit.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash + +# Fail closed before a release job builds a commit that main never accepted or +# that CI never validated. +# +# check-release-state.sh answers "is this tag safe to write to". This answers +# the other half: "is this commit safe to ship". The release workflow already +# proves the tag resolves to the checkout, but a tag can be pushed to any +# commit at any time, including one that never landed on main and one whose +# tests have not finished. + +set -euo pipefail + +release_commit="${1:-}" +fixture="${2:-}" +repository="${GITHUB_REPOSITORY:-pgsty/silo}" +release_branch="${RELEASE_BRANCH:-origin/main}" + +# Workflows that must have a successful run for this exact commit, keyed by the +# workflow file path rather than the display name: a name is free text, so a +# second workflow could otherwise be added that calls itself "Go CI", succeeds +# trivially, and satisfies this check while the real one fails. +required_workflow_paths=( + ".github/workflows/go.yml" + ".github/workflows/vulncheck.yml" +) + +if ! command -v jq >/dev/null 2>&1; then + echo "jq is required to inspect workflow runs" >&2 + exit 1 +fi + +if [[ ! "${release_commit}" =~ ^[0-9a-f]{40}$ ]]; then + echo "Invalid release commit: ${release_commit:-}" >&2 + exit 1 +fi + +if [ -z "${fixture}" ]; then + if ! git rev-parse --verify --quiet "${release_branch}" >/dev/null; then + echo "Cannot resolve ${release_branch}; fetch it before releasing" >&2 + exit 1 + fi + # The exact tip, not merely an ancestor: an older commit on main has no + # push run of its own when it arrived as part of a multi-commit push, and a + # release must not trail behind what main already contains. + tip="$(git rev-parse "${release_branch}^{commit}")" + if [ "${tip}" != "${release_commit}" ]; then + echo "Refusing to release ${release_commit}: ${release_branch} is at ${tip}" >&2 + exit 1 + fi + echo "Release commit ${release_commit} is the tip of ${release_branch}." + + error_file="$(mktemp)" + trap 'rm -f "${error_file}"' EXIT + # A pull_request run checks out GitHub's synthetic merge ref, so a green + # run proves the merge result was good, not that this commit was ever built + # on its own. exclude_pull_requests only trims the pull_requests field of + # each run; the event/head_branch filter below is what drops those runs. + if ! runs_json="$( + gh api --paginate \ + "repos/${repository}/actions/runs?head_sha=${release_commit}&exclude_pull_requests=true&per_page=100" \ + --jq '.workflow_runs[] | {path, event, head_branch, head_sha, status, conclusion, run_number, run_attempt}' \ + 2>"${error_file}" | jq -s '.' + )"; then + cat "${error_file}" >&2 + exit 1 + fi +else + runs_json="$(<"${fixture}")" +fi + +if ! jq -e 'type == "array" and all(.[]; type == "object" and (.path | type == "string"))' \ + <<<"${runs_json}" >/dev/null 2>&1; then + echo "Invalid workflow run response for ${release_commit}" >&2 + exit 1 +fi + +fail=0 +for workflow in "${required_workflow_paths[@]}"; do + # Only runs that actually built this commit count: the right workflow file, + # triggered by the push that put this SHA on main. A manual dispatch is not + # substitute evidence - it can be run from any ref. Take the newest such + # run; an older success must not paper over a later failure. + latest="$(jq -c --arg path "${workflow}" --arg sha "${release_commit}" ' + [ .[] + | select(.path == $path) + | select(.head_sha == $sha) + | select(.event == "push" and .head_branch == "main") + ] + | sort_by([.run_number // 0, .run_attempt // 0]) + | last // null' <<<"${runs_json}")" + + if [ "${latest}" = "null" ]; then + echo "Required workflow ${workflow} has no qualifying run for ${release_commit} (no run)" >&2 + fail=1 + continue + fi + + status="$(jq -r '.status // "unknown"' <<<"${latest}")" + conclusion="$(jq -r '.conclusion // "none"' <<<"${latest}")" + if [ "${status}" != "completed" ] || [ "${conclusion}" != "success" ]; then + observed="${conclusion}" + if [ "${status}" != "completed" ]; then + observed="${status}" + fi + echo "Required workflow ${workflow} did not succeed for ${release_commit} (${observed})" >&2 + fail=1 + fi +done + +if [ "${fail}" -ne 0 ]; then + echo "Refusing to release ${release_commit}: required checks are not green" >&2 + exit 1 +fi + +echo "Required workflows are green for ${release_commit}." diff --git a/buildscripts/check-release-commit_test.sh b/buildscripts/check-release-commit_test.sh new file mode 100755 index 000000000..86dba366b --- /dev/null +++ b/buildscripts/check-release-commit_test.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +checker="${script_dir}/check-release-commit.sh" +commit="0123456789abcdef0123456789abcdef01234567" +other_commit="89abcdef0123456789abcdef0123456789abcdef" +fixture="$(mktemp)" +stdout_file="$(mktemp)" +stderr_file="$(mktemp)" +trap 'rm -f "${fixture}" "${stdout_file}" "${stderr_file}"' EXIT + +expect_success() { + if ! "${checker}" "$@" >"${stdout_file}" 2>"${stderr_file}"; then + cat "${stderr_file}" >&2 + return 1 + fi +} + +expect_failure() { + if "${checker}" "$@" >"${stdout_file}" 2>"${stderr_file}"; then + echo "Expected release-commit check to fail: $*" >&2 + cat "${stdout_file}" >&2 + return 1 + fi +} + +# run [event] [branch] [sha] [run_number] [status] +run() { + local path="$1" conclusion="$2" + local event="${3:-push}" branch="${4:-main}" sha="${5:-${commit}}" + local number="${6:-1}" status="${7:-completed}" + local conclusion_json="\"${conclusion}\"" + [ "${conclusion}" = "null" ] && conclusion_json=null + printf '{"path":"%s","event":"%s","head_branch":"%s","head_sha":"%s","status":"%s","conclusion":%s,"run_number":%s,"run_attempt":1}' \ + "${path}" "${event}" "${branch}" "${sha}" "${status}" "${conclusion_json}" "${number}" +} + +go_ok="$(run .github/workflows/go.yml success)" +vuln_ok="$(run .github/workflows/vulncheck.yml success)" + +all_green() { + printf '[%s,%s]\n' "${go_ok}" "${vuln_ok}" >"${fixture}" +} + +# Baseline: both required workflows green on a push to main. +all_green +expect_success "${commit}" "${fixture}" +grep -qF "Required workflows are green for ${commit}." "${stdout_file}" + +# A manual dispatch is not evidence: it can be run from any ref. +printf '[%s,%s]\n' \ + "$(run .github/workflows/go.yml success workflow_dispatch main)" \ + "${vuln_ok}" >"${fixture}" +expect_failure "${commit}" "${fixture}" +grep -qF "no qualifying run" "${stderr_file}" + +# A re-run that finally succeeded is enough; the earlier failure is history. +printf '[%s,%s,%s]\n' \ + "$(run .github/workflows/go.yml failure push main "${commit}" 1)" \ + "$(run .github/workflows/go.yml success push main "${commit}" 2)" \ + "${vuln_ok}" >"${fixture}" +expect_success "${commit}" "${fixture}" + +# ...but a LATER failure must not be papered over by an earlier success. +printf '[%s,%s,%s]\n' \ + "$(run .github/workflows/go.yml success push main "${commit}" 1)" \ + "$(run .github/workflows/go.yml failure push main "${commit}" 2)" \ + "${vuln_ok}" >"${fixture}" +expect_failure "${commit}" "${fixture}" +grep -qF "Required workflow .github/workflows/go.yml did not succeed" "${stderr_file}" +grep -qF "(failure)" "${stderr_file}" + +# A decoy workflow that merely calls itself "Go CI" must not satisfy the gate. +printf '[%s,%s]\n' \ + "$(run .github/workflows/decoy.yml success)" "${vuln_ok}" >"${fixture}" +expect_failure "${commit}" "${fixture}" +grep -qF "Required workflow .github/workflows/go.yml has no qualifying run" "${stderr_file}" + +# A pull_request run tests GitHub's merge ref, not this commit on its own. +printf '[%s,%s]\n' \ + "$(run .github/workflows/go.yml success pull_request feature)" "${vuln_ok}" >"${fixture}" +expect_failure "${commit}" "${fixture}" +grep -qF "no qualifying run" "${stderr_file}" + +# A push run on some other branch is not evidence that main accepted it. +printf '[%s,%s]\n' \ + "$(run .github/workflows/go.yml success push feature)" "${vuln_ok}" >"${fixture}" +expect_failure "${commit}" "${fixture}" +grep -qF "no qualifying run" "${stderr_file}" + +# A run reporting a different head_sha must be ignored. +printf '[%s,%s]\n' \ + "$(run .github/workflows/go.yml success push main "${other_commit}")" "${vuln_ok}" >"${fixture}" +expect_failure "${commit}" "${fixture}" +grep -qF "no qualifying run" "${stderr_file}" + +# Missing workflow entirely. +printf '[%s]\n' "${go_ok}" >"${fixture}" +expect_failure "${commit}" "${fixture}" +grep -qF "Required workflow .github/workflows/vulncheck.yml has no qualifying run" "${stderr_file}" + +# Still running. +printf '[%s,%s]\n' "${go_ok}" \ + "$(run .github/workflows/vulncheck.yml null push main "${commit}" 1 in_progress)" >"${fixture}" +expect_failure "${commit}" "${fixture}" +grep -qF "(in_progress)" "${stderr_file}" + +# Cancelled counts as not green. +printf '[%s,%s]\n' "${go_ok}" \ + "$(run .github/workflows/vulncheck.yml cancelled)" >"${fixture}" +expect_failure "${commit}" "${fixture}" +grep -qF "(cancelled)" "${stderr_file}" + +# Malformed API response. +printf '{not-json}\n' >"${fixture}" +expect_failure "${commit}" "${fixture}" +grep -qF "Invalid workflow run response for ${commit}" "${stderr_file}" + +# An empty array is a valid response that satisfies nothing. +printf '[]\n' >"${fixture}" +expect_failure "${commit}" "${fixture}" +grep -qF "Required workflow .github/workflows/go.yml has no qualifying run" "${stderr_file}" + +expect_failure "not-a-commit" "${fixture}" +grep -qF "Invalid release commit" "${stderr_file}" + +expect_failure "" "${fixture}" +grep -qF "Invalid release commit: " "${stderr_file}" + +echo "release-commit decision tests passed" diff --git a/buildscripts/checkdeps.sh b/buildscripts/checkdeps.sh index da570c13f..381b4fead 100755 --- a/buildscripts/checkdeps.sh +++ b/buildscripts/checkdeps.sh @@ -7,7 +7,7 @@ _init() { ## Minimum required versions for build dependencies GIT_VERSION="1.0" - GO_VERSION="1.27.3" + GO_VERSION="1.27.4" OSX_VERSION="10.8" KNAME=$(uname -s) ARCH=$(uname -m) diff --git a/buildscripts/install-mcli.sh b/buildscripts/install-mcli.sh index f72e2ad8c..7a19e1409 100755 --- a/buildscripts/install-mcli.sh +++ b/buildscripts/install-mcli.sh @@ -40,7 +40,7 @@ if [ -n "${MCLI_BIN:-}" ]; then exit 0 fi -release=${MCLI_RELEASE:-RELEASE.2026-11-09T00-00-00Z} +release=${MCLI_RELEASE:-RELEASE.2026-11-12T00-00-00Z} version_hyphen=${release#RELEASE.} package_version=$(printf '%s\n' "${version_hyphen}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/') if [ "${package_version}" = "${version_hyphen}" ]; then diff --git a/buildscripts/release-dependency-guard/main.go b/buildscripts/release-dependency-guard/main.go new file mode 100644 index 000000000..ccf5ea0ba --- /dev/null +++ b/buildscripts/release-dependency-guard/main.go @@ -0,0 +1,156 @@ +// Copyright (c) 2026 PGSTY +// SPDX-License-Identifier: AGPL-3.0-or-later + +// release-dependency-guard checks the modules and bundled client actually used +// by a release. A local workspace must not substitute unreleased source trees. +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + + "golang.org/x/mod/semver" +) + +type module struct { + Path, Version, Dir string + Main bool + Replace *module +} + +func selected(m module, path string) (module, error) { + if m.Replace != nil { + m = *m.Replace + } + if m.Main || m.Path != path || m.Version == "" { + return module{}, fmt.Errorf("%s must select an immutable maintained module, got %s@%s", path, m.Path, m.Version) + } + return m, nil +} + +func atLeast(version, minimum string) bool { + return semver.IsValid(version) && semver.Prerelease(version) == "" && semver.Compare(version, minimum) >= 0 +} + +func validate(modules map[string]module, consoleVersion, mcTag, mcCommit string) error { + pkg, err := selected(modules["github.com/pgsty/silo-pkg/v3"], "github.com/pgsty/silo-pkg/v3") + if err != nil { + return err + } + if !atLeast(pkg.Version, "v3.16.1") { + return fmt.Errorf("silo-pkg %s lacks the strict policy release; need v3.16.1 or newer", pkg.Version) + } + if _, err := selected(modules["github.com/minio/console"], "github.com/pgsty/silo-console"); err != nil { + return err + } + if !atLeast("v"+consoleVersion, "v2.7.1") { + return fmt.Errorf("embedded Console %s lacks the policy fixes; need 2.7.1 or newer", consoleVersion) + } + mc, err := selected(modules["github.com/minio/mc"], "github.com/pgsty/mc") + if err != nil { + return err + } + if !regexp.MustCompile(`^RELEASE\.\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}Z$`).MatchString(mcTag) || mcTag <= "RELEASE.2026-11-09T00-00-00Z" { + return fmt.Errorf("bundled mcli %s must be a release after 20261109", mcTag) + } + if len(mcCommit) != 40 || !strings.HasSuffix(mc.Version, "-"+mcCommit[:12]) { + return fmt.Errorf("mc module %s does not match Dockerfile.goreleaser tag %s (%s)", mc.Version, mcTag, mcCommit) + } + return nil +} + +func command(name string, args ...string) ([]byte, error) { + cmd := exec.Command(name, args...) + cmd.Stderr = os.Stderr + return cmd.Output() +} + +func check() error { + if err := os.Setenv("GOWORK", "off"); err != nil { + return err + } + data, err := command("go", "list", "-mod=readonly", "-m", "-json", "github.com/pgsty/silo-pkg/v3", "github.com/minio/console", "github.com/minio/mc") + if err != nil { + return err + } + modules := make(map[string]module) + dec := json.NewDecoder(bytes.NewReader(data)) + for { + var m module + if err := dec.Decode(&m); err != nil { + if err == io.EOF { + break + } + return err + } + modules[m.Path] = m + } + pkg, err := selected(modules["github.com/pgsty/silo-pkg/v3"], "github.com/pgsty/silo-pkg/v3") + if err != nil { + return err + } + if !atLeast(pkg.Version, "v3.16.1") { + return fmt.Errorf("release blocked: select silo-pkg v3.16.1 or newer (currently %s)", pkg.Version) + } + console, err := selected(modules["github.com/minio/console"], "github.com/pgsty/silo-console") + if err != nil { + return err + } + if console.Dir == "" { + data, err = command("go", "mod", "download", "-json", console.Path+"@"+console.Version) + if err != nil { + return err + } + if err := json.Unmarshal(data, &console); err != nil { + return err + } + } + data, err = os.ReadFile(filepath.Join(console.Dir, "web-app", "package.json")) + if err != nil { + return err + } + var metadata struct{ Version string } + if err := json.Unmarshal(data, &metadata); err != nil { + return err + } + data, err = os.ReadFile("Dockerfile.goreleaser") + if err != nil { + return err + } + match := regexp.MustCompile(`(?m)^ARG MC_VERSION=(\S+)\s*$`).FindSubmatch(data) + if len(match) != 2 { + return fmt.Errorf("Dockerfile.goreleaser must pin MC_VERSION") + } + tag := string(match[1]) + data, err = command("gh", "api", "repos/pgsty/mc/commits/"+tag, "--jq", ".sha") + if err != nil { + return err + } + commit := strings.TrimSpace(string(data)) + if err := validate(modules, metadata.Version, tag, commit); err != nil { + return err + } + data, err = command("gh", "api", "repos/pgsty/mc/compare/d7262c2a5fbdf2b9503cc0283f1d37ae8da41baf..."+commit, "--jq", ".status") + if err != nil { + return err + } + if status := strings.TrimSpace(string(data)); status != "ahead" && status != "identical" { + return fmt.Errorf("mcli %s does not include the access-key policy repair", tag) + } + fmt.Printf("Release dependencies verified: silo-pkg %s, Console %s, mcli %s (%s)\n", pkg.Version, metadata.Version, tag, commit) + return nil +} + +func main() { + if err := check(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/buildscripts/release-dependency-guard/main_test.go b/buildscripts/release-dependency-guard/main_test.go new file mode 100644 index 000000000..3d6877517 --- /dev/null +++ b/buildscripts/release-dependency-guard/main_test.go @@ -0,0 +1,44 @@ +// Copyright (c) 2026 PGSTY +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import "testing" + +func TestReleaseDependencySelection(t *testing.T) { + const commit = "0123456789abcdef0123456789abcdef01234567" + for _, tc := range []struct { + name, pkg, console, tag, mcSHA string + local, upstream, wantError bool + }{ + {name: "coordinated", pkg: "v3.16.1", console: "2.7.1", tag: "RELEASE.2026-11-12T00-00-00Z", mcSHA: commit}, + {name: "old-package", pkg: "v3.16.0", console: "2.7.1", tag: "RELEASE.2026-11-12T00-00-00Z", mcSHA: commit, wantError: true}, + {name: "prerelease-package", pkg: "v3.16.1-rc.1", console: "2.7.1", tag: "RELEASE.2026-11-12T00-00-00Z", mcSHA: commit, wantError: true}, + {name: "old-embedded-console", pkg: "v3.16.1", console: "2.7.0", tag: "RELEASE.2026-11-12T00-00-00Z", mcSHA: commit, wantError: true}, + {name: "old-container-client", pkg: "v3.16.1", console: "2.7.1", tag: "RELEASE.2026-11-09T00-00-00Z", mcSHA: commit, wantError: true}, + {name: "source-binary-mismatch", pkg: "v3.16.1", console: "2.7.1", tag: "RELEASE.2026-11-12T00-00-00Z", mcSHA: "ffffffffffffffffffffffffffffffffffffffff", wantError: true}, + {name: "workspace-console", pkg: "v3.16.1", console: "2.7.1", tag: "RELEASE.2026-11-12T00-00-00Z", mcSHA: commit, local: true, wantError: true}, + {name: "upstream-console", pkg: "v3.16.1", console: "2.7.1", tag: "RELEASE.2026-11-12T00-00-00Z", mcSHA: commit, upstream: true, wantError: true}, + } { + t.Run(tc.name, func(t *testing.T) { + modules := map[string]module{ + "github.com/pgsty/silo-pkg/v3": {Path: "github.com/pgsty/silo-pkg/v3", Version: tc.pkg}, + "github.com/minio/console": {Path: "github.com/minio/console", Replace: &module{ + Path: "github.com/pgsty/silo-console", Version: "v0.0.0-20261112000000-0123456789ab", + }}, + "github.com/minio/mc": {Path: "github.com/minio/mc", Replace: &module{ + Path: "github.com/pgsty/mc", Version: "v0.0.0-20261112000000-0123456789ab", + }}, + } + if tc.local { + modules["github.com/minio/console"].Replace.Version = "" + } + if tc.upstream { + modules["github.com/minio/console"].Replace.Path = "github.com/minio/console" + } + if err := validate(modules, tc.console, tc.tag, tc.mcSHA); (err != nil) != tc.wantError { + t.Fatalf("got %v, wantError=%t", err, tc.wantError) + } + }) + } +} diff --git a/cmd/admin-bucket-handlers.go b/cmd/admin-bucket-handlers.go index 308aa955c..2feb0ddc7 100644 --- a/cmd/admin-bucket-handlers.go +++ b/cmd/admin-bucket-handlers.go @@ -817,17 +817,17 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r * bucketMap[bucket] = &v } - if globalSiteReplicationSys.isEnabled() && v.Suspended() { - rpt.SetStatus(bucket, fileName, fmt.Errorf("Cluster replication is enabled for this site, so the versioning state cannot be suspended.")) + if globalSiteReplicationSys.isEnabled() && (v.Suspended() || v.PrefixesExcluded()) { + rpt.SetStatus(bucket, fileName, fmt.Errorf("Cluster replication is enabled for this site, so the versioning state cannot be suspended or exclude prefixes.")) continue } - if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled && v.Suspended() { - rpt.SetStatus(bucket, fileName, fmt.Errorf("An Object Lock configuration is present on this bucket, so the versioning state cannot be suspended.")) + if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled && (v.Suspended() || v.PrefixesExcluded()) { + rpt.SetStatus(bucket, fileName, fmt.Errorf("An Object Lock configuration is present on this bucket, so the versioning state cannot be suspended or exclude prefixes.")) continue } - if rcfg, _ := getReplicationConfig(ctx, bucket); rcfg != nil && v.Suspended() { - rpt.SetStatus(bucket, fileName, fmt.Errorf("A replication configuration is present on this bucket, so the versioning state cannot be suspended.")) + if rcfg, _ := getReplicationConfig(ctx, bucket); rcfg != nil && (v.Suspended() || v.PrefixesExcluded()) { + rpt.SetStatus(bucket, fileName, fmt.Errorf("A replication configuration is present on this bucket, so the versioning state cannot be suspended or exclude prefixes.")) continue } @@ -907,7 +907,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r * continue } - bucketPolicy, err := policy.ParseBucketPolicyConfig(bytes.NewReader(bucketPolicyBytes), bucket) + bucketPolicy, err := parseBucketPolicyStrictBytes(bucketPolicyBytes, bucket) if err != nil { rpt.SetStatus(bucket, fileName, err) continue diff --git a/cmd/admin-handlers-site-replication.go b/cmd/admin-handlers-site-replication.go index 3be2dee76..422e3a766 100644 --- a/cmd/admin-handlers-site-replication.go +++ b/cmd/admin-handlers-site-replication.go @@ -165,10 +165,10 @@ func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http. default: err = errSRInvalidRequest(errInvalidArgument) case madmin.SRIAMItemPolicy: - if item.Policy == nil { + if item.Policy == nil || bytes.Equal(bytes.TrimSpace(item.Policy), []byte("null")) { err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt) } else { - policy, perr := policy.ParseConfig(bytes.NewReader(item.Policy)) + policy, perr := parsePolicySnapshotBytes(item.Policy) if perr != nil { writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, perr), r.URL) return @@ -222,10 +222,10 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht default: err = globalSiteReplicationSys.PeerBucketMetadataUpdateHandler(ctx, item) case madmin.SRBucketMetaTypePolicy: - if item.Policy == nil { + if item.Policy == nil || bytes.Equal(bytes.TrimSpace(item.Policy), []byte("null")) { err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil, item.UpdatedAt) } else { - bktPolicy, berr := policy.ParseBucketPolicyConfig(bytes.NewReader(item.Policy), item.Bucket) + bktPolicy, berr := parseBucketPolicyStrictBytes(item.Policy, item.Bucket) if berr != nil { writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, berr), r.URL) return diff --git a/cmd/admin-handlers-users.go b/cmd/admin-handlers-users.go index 8d82b846a..079e92d61 100644 --- a/cmd/admin-handlers-users.go +++ b/cmd/admin-handlers-users.go @@ -889,8 +889,9 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re // policy the account has. An empty policy document is passed on as // such: it is the request to remove the embedded policy. var sp *policy.Policy - if len(updateReq.NewPolicy) > 0 && !bytes.Equal(updateReq.NewPolicy, []byte("null")) { - sp, err = policy.ParseConfigStrict(bytes.NewReader(updateReq.NewPolicy)) + trimmedPolicy := bytes.TrimSpace(updateReq.NewPolicy) + if len(trimmedPolicy) > 0 && !bytes.Equal(trimmedPolicy, []byte("null")) { + sp, err = parseSessionPolicyStrictBytes(trimmedPolicy) if err != nil { writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) return @@ -1766,7 +1767,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request return } - iamPolicy, err := policy.ParseConfigStrict(bytes.NewReader(iamPolicyBytes)) + iamPolicy, err := parsePolicyStrictBytes(iamPolicyBytes) if err != nil { writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) return @@ -2510,23 +2511,34 @@ func (a adminAPIHandlers) importIAM(w http.ResponseWriter, r *http.Request, apiV return default: defer f.Close() - var allPolicies map[string]policy.Policy + var policyDocuments map[string]json.RawMessage data, err = io.ReadAll(f) if err != nil { writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allPoliciesFile, ""), r.URL) return } - err = json.Unmarshal(data, &allPolicies) + err = json.Unmarshal(data, &policyDocuments) if err != nil { writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminConfigBadJSON, err, allPoliciesFile, ""), r.URL) return } + // Validate the entire policy file before applying any entries. Keep + // legacy semantic validation, but never discard unknown wire fields. + allPolicies := make(map[string]*policy.Policy, len(policyDocuments)) + for policyName, document := range policyDocuments { + parsed, parseErr := parsePolicySnapshotBytes(document) + if parseErr != nil { + writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminConfigBadJSON, parseErr, allPoliciesFile, policyName), r.URL) + return + } + allPolicies[policyName] = parsed + } for policyName, policy := range allPolicies { if policy.IsEmpty() { err = globalIAMSys.DeletePolicy(ctx, policyName, true) removed.Policies = append(removed.Policies, policyName) } else { - _, err = globalIAMSys.SetPolicy(ctx, policyName, policy) + _, err = globalIAMSys.SetPolicy(ctx, policyName, *policy) added.Policies = append(added.Policies, policyName) } if err != nil { @@ -2674,10 +2686,13 @@ func (a adminAPIHandlers) importIAM(w http.ResponseWriter, r *http.Request, apiV var sp *policy.Policy var err error if len(svcAcctReq.SessionPolicy) > 0 { - sp, err = policy.ParseConfig(bytes.NewReader(svcAcctReq.SessionPolicy)) - if err != nil { - writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL) - return + trimmedSP := bytes.TrimSpace(svcAcctReq.SessionPolicy) + if len(trimmedSP) > 0 && !bytes.Equal(trimmedSP, []byte("null")) { + sp, err = parsePolicySnapshotBytes(trimmedSP) + if err != nil { + writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL) + return + } } } // service account access key cannot have space characters @@ -2706,15 +2721,16 @@ func (a adminAPIHandlers) importIAM(w http.ResponseWriter, r *http.Request, apiV } } opts := newServiceAccountOpts{ - accessKey: user, - secretKey: svcAcctReq.SecretKey, - status: svcAcctReq.Status, - sessionPolicy: sp, - claims: svcAcctReq.Claims, - name: svcAcctReq.Name, - description: svcAcctReq.Description, - expiration: svcAcctReq.Expiration, - allowSiteReplicatorAccount: false, + accessKey: user, + secretKey: svcAcctReq.SecretKey, + status: svcAcctReq.Status, + sessionPolicy: sp, + claims: svcAcctReq.Claims, + name: svcAcctReq.Name, + description: svcAcctReq.Description, + expiration: svcAcctReq.Expiration, + allowSiteReplicatorAccount: false, + preserveEmptyEmbeddedPolicy: svcAcctClaimsHaveEmbeddedPolicy(svcAcctReq.Claims), } if _, _, err = globalIAMSys.NewServiceAccount(ctx, svcAcctReq.Parent, svcAcctReq.Groups, opts); err != nil { @@ -3058,8 +3074,11 @@ func commonAddServiceAccount(r *http.Request, ldap bool) (context.Context, auth. } var sp *policy.Policy - if len(createReq.Policy) > 0 { - sp, err = policy.ParseConfigStrict(bytes.NewReader(createReq.Policy)) + trimmedCreatePolicy := bytes.TrimSpace(createReq.Policy) + if bytes.Equal(trimmedCreatePolicy, []byte("null")) { + createReq.Policy = nil + } else if len(trimmedCreatePolicy) > 0 { + sp, err = parseSessionPolicyStrictBytes(trimmedCreatePolicy) if err != nil { return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", toAdminAPIErr(ctx, err) } diff --git a/cmd/api-response.go b/cmd/api-response.go index 27749b9f5..210077063 100644 --- a/cmd/api-response.go +++ b/cmd/api-response.go @@ -568,18 +568,15 @@ func generateListVersionsResponse(ctx context.Context, bucket, prefix, marker, v DisplayName: "minio", } data := ListVersionsResponse{} - var lastObjMetaName string var tagErr, metaErr APIErrorCode = -1, -1 for _, object := range resp.Objects { if object.Name == "" { continue } - // Cache checks for the same object - if metadata != nil && lastObjMetaName != object.Name { - tagErr = metadata(object.Name, policy.GetObjectTaggingAction) - metaErr = metadata(object.Name, policy.GetObjectAction) - lastObjMetaName = object.Name + if metadata != nil { + tagErr = metadata(object, policy.GetObjectTaggingAction) + metaErr = metadata(object, policy.GetObjectAction) } content := ObjectVersion{} content.Key = s3EncodeName(object.Name, encodingType) @@ -727,10 +724,10 @@ func generateListObjectsV2Response(ctx context.Context, bucket, prefix, token, n } content.Owner = owner if metadata != nil { - if metadata(object.Name, policy.GetObjectTaggingAction) == ErrNone { + if metadata(object, policy.GetObjectTaggingAction) == ErrNone { content.UserTags = object.UserTags } - if metadata(object.Name, policy.GetObjectAction) == ErrNone { + if metadata(object, policy.GetObjectAction) == ErrNone { content.UserMetadata = &Metadata{} switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind { case crypto.S3: @@ -774,7 +771,7 @@ func generateListObjectsV2Response(ctx context.Context, bucket, prefix, token, n return data } -type metaCheckFn = func(name string, action policy.Action) (s3Err APIErrorCode) +type metaCheckFn = func(object ObjectInfo, action policy.Action) (s3Err APIErrorCode) // generates CopyObjectResponse from the committed object information. func generateCopyObjectResponse(oi ObjectInfo, cs map[string]string) CopyObjectResponse { diff --git a/cmd/auth-handler-unread-tags.go b/cmd/auth-handler-unread-tags.go index b3b95f894..37d43aecc 100644 --- a/cmd/auth-handler-unread-tags.go +++ b/cmd/auth-handler-unread-tags.go @@ -35,6 +35,349 @@ import ( "github.com/puzpuzpuz/xsync/v3" ) +func withScopedReqObject(ctx context.Context, bucket, object, versionID string, fn func() APIErrorCode) APIErrorCode { + reqInfo := logger.GetReqInfo(ctx) + if reqInfo == nil { + return ErrAccessDenied + } + prevBucket, prevObject, prevVID := reqInfo.BucketName, reqInfo.ObjectName, reqInfo.VersionID + reqInfo.BucketName = bucket + reqInfo.ObjectName = object + reqInfo.VersionID = versionID + defer func() { + reqInfo.BucketName = prevBucket + reqInfo.ObjectName = prevObject + reqInfo.VersionID = prevVID + }() + return fn() +} + +func objectVersionAction(base policy.Action, versionID string) (policy.Action, bool) { + if versionID == "" { + return base, false + } + switch base { + case policy.GetObjectAction: + return policy.GetObjectVersionAction, true + case policy.GetObjectTaggingAction: + return policy.GetObjectVersionTaggingAction, true + case policy.PutObjectTaggingAction: + return policy.PutObjectVersionTaggingAction, true + case policy.DeleteObjectTaggingAction: + return policy.DeleteObjectVersionTaggingAction, true + default: + return base, false + } +} + +func (sys *IAMSys) policySetsForExplicitDeny(cred auth.Credentials) [][]policy.Policy { + derived := cred.IsTemp() || cred.IsServiceAccount() + name := cred.AccessKey + if derived { + name = cred.ParentUser + } + cache := sys.store.rlock() + var names []string + roleARN := (policy.Args{Claims: cred.Claims}).GetRoleArn() + switch { + case name == globalActiveCred.AccessKey: + case derived && roleARN != "": + parsed, err := arn.Parse(roleARN) + if err != nil { + sys.store.runlock() + return nil + } + names = newMappedPolicy(sys.rolesMap[parsed]).toSlice() + default: + var err error + names, _, err = cache.policyDBGet(sys.store, name, false, false) + if err != nil && err != errNoSuchPolicy { + sys.store.runlock() + return nil + } + groups, gerr := cache.policyDBGetGroups(sys.store, len(names) > 0, cred.Groups...) + if gerr != nil && gerr != errNoSuchPolicy { + sys.store.runlock() + return nil + } + names = append(names, groups...) + if derived && len(names) == 0 { + claims, _ := policy.GetPoliciesFromClaims(cred.Claims, iamPolicyClaimNameOpenID()) + names = claims.ToSlice() + } + } + sys.store.runlock() + + sets := [][]policy.Policy{nil} + slices.Sort(names) + compactNames := slices.Compact(names) + if len(compactNames) > 0 { + if avail, merged, err := sys.store.MergePolicies(strings.Join(compactNames, ",")); err == nil && avail != "" { + sets[0] = append(sets[0], merged) + } + } + if derived && (cred.IsTemp() || cred.Claims[iamPolicyClaimNameSA()] != inheritedPolicyType) { + if sp, ok := cred.Claims[sessionPolicyNameExtracted].(string); ok { + if p, err := policy.ParseConfig(strings.NewReader(sp)); err == nil { + sets = append(sets, []policy.Policy{*p}) + } + } + } + return sets +} + +func isActionExplicitlyDenied(ctx context.Context, r *http.Request, action policy.Action, existingTags string, requestTags *string) bool { + reqInfo := logger.GetReqInfo(ctx) + if reqInfo == nil { + return true + } + if reqInfo.Owner { + return false + } + cred := reqInfo.Cred + bucket := reqInfo.BucketName + object := reqInfo.ObjectName + versionID := reqInfo.VersionID + buildValues := func(loc string, c auth.Credentials) map[string][]string { + values := getConditionValuesWithTags(r, loc, c, existingTags, requestTags) + if actionUsesObjectVersion(action) { + if versionID == "" { + delete(values, "versionid") + } else { + values["versionid"] = []string{versionID} + } + } + return values + } + + if cred.AccessKey == "" { + p, err := globalPolicySys.Get(bucket) + if err != nil { + return false + } + bArgs := policy.BucketPolicyArgs{ + Action: action, + BucketName: bucket, + ObjectName: object, + ConditionValues: buildValues(reqInfo.Region, auth.AnonymousCredentials), + IsOwner: false, + } + for _, st := range p.Statements { + if st.Effect != policy.Deny { + continue + } + probe := st + probe.Effect = policy.Allow + if probe.IsAllowed(bArgs) { + return true + } + } + return false + } + + values := buildValues("", cred) + if newGlobalAuthZPluginFn() != nil { + return !globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: action, + BucketName: bucket, + ObjectName: object, + ConditionValues: values, + IsOwner: false, + Claims: cred.Claims, + DenyOnly: true, + }) + } + if !globalIAMSys.Initialized() { + return false + } + args := policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: action, + BucketName: bucket, + ObjectName: object, + ConditionValues: values, + IsOwner: false, + Claims: cred.Claims, + DenyOnly: true, + } + for _, policies := range globalIAMSys.policySetsForExplicitDeny(cred) { + if len(policies) == 0 { + continue + } + merged := policy.MergePolicies(policies...) + if !merged.IsAllowed(args) { + return true + } + } + return false +} + +func authorizeVersionedRequestWithTags(ctx context.Context, r *http.Request, baseAction policy.Action, versionID, existingTags string, requestTags *string) APIErrorCode { + reqInfo := logger.GetReqInfo(ctx) + if reqInfo == nil { + return ErrAccessDenied + } + prevVID := reqInfo.VersionID + reqInfo.VersionID = versionID + defer func() { reqInfo.VersionID = prevVID }() + + if verAction, ok := objectVersionAction(baseAction, versionID); ok { + if authorizeRequestWithTags(ctx, r, verAction, existingTags, requestTags) == ErrNone { + return ErrNone + } + if isActionExplicitlyDenied(ctx, r, verAction, existingTags, requestTags) { + return ErrAccessDenied + } + } + return authorizeRequestWithTags(ctx, r, baseAction, existingTags, requestTags) +} + +func authorizeVersionedRequestWithExistingTags(ctx context.Context, r *http.Request, baseAction policy.Action, versionID, existingTags string) APIErrorCode { + return authorizeVersionedRequestWithTags(ctx, r, baseAction, versionID, existingTags, nil) +} + +func authorizeVersionedRequestWithUnreadTags(ctx context.Context, r *http.Request, baseAction policy.Action, versionID string, readErr error) APIErrorCode { + s3Err := authorizeVersionedRequestWithExistingTags(ctx, r, baseAction, versionID, "") + if s3Err == ErrNone || toAPIError(ctx, readErr).HTTPStatusCode < http.StatusInternalServerError { + return s3Err + } + return authorizeVersionedRequestWithTagCandidates(ctx, r, baseAction, versionID, s3Err) +} + +func authorizeVersionedRequestWithPotentialTags(ctx context.Context, r *http.Request, baseAction policy.Action, versionID string) APIErrorCode { + s3Err := authorizeVersionedRequestWithExistingTags(ctx, r, baseAction, versionID, "") + if s3Err == ErrNone { + return ErrNone + } + return authorizeVersionedRequestWithTagCandidates(ctx, r, baseAction, versionID, s3Err) +} + +func authorizeVersionedRequestWithTagCandidates(ctx context.Context, r *http.Request, baseAction policy.Action, versionID string, s3Err APIErrorCode) APIErrorCode { + reqInfo := logger.GetReqInfo(ctx) + if reqInfo == nil { + return ErrAccessDenied + } + prevVID := reqInfo.VersionID + reqInfo.VersionID = versionID + defer func() { reqInfo.VersionID = prevVID }() + + candidates := policyExistingObjectTags(ctx, r, baseAction) + if verAction, ok := objectVersionAction(baseAction, versionID); ok { + candidates = append(candidates, policyExistingObjectTags(ctx, r, verAction)...) + } + for _, existingTags := range slices.Compact(candidates) { + if authorizeVersionedRequestWithExistingTags(ctx, r, baseAction, versionID, existingTags) == ErrNone { + return ErrNone + } + } + return s3Err +} + +// canPotentiallyAllowAction reports whether the authenticated caller has any +// grant capable of allowing action on bucket/object once condition values +// (such as copied source tags on CopyObject) are resolved. +func canPotentiallyAllowAction(ctx context.Context, r *http.Request, action policy.Action, bucket, object string) bool { + reqInfo := logger.GetReqInfo(ctx) + if reqInfo == nil { + return false + } + emptyTags := "" + if withScopedReqObject(ctx, bucket, object, "", func() APIErrorCode { + return authorizeRequestWithTags(ctx, r, action, "", &emptyTags) + }) == ErrNone { + return true + } + if reqInfo.Owner { + return true + } + cred := reqInfo.Cred + if cred.AccessKey == "" { + p, err := globalPolicySys.Get(bucket) + if err != nil { + return false + } + values := getConditionValues(r, reqInfo.Region, auth.AnonymousCredentials) + for _, st := range p.Statements { + if st.Effect != policy.Allow { + continue + } + probe := st + probe.Conditions = nil + if probe.IsAllowed(policy.BucketPolicyArgs{Action: action, BucketName: bucket, ObjectName: object, ConditionValues: values}) { + return true + } + } + return false + } + if newGlobalAuthZPluginFn() != nil || !globalIAMSys.Initialized() { + return true + } + values := getConditionValues(r, "", cred) + for _, policies := range globalIAMSys.policySetsForExplicitDeny(cred) { + setAllows := false + for _, p := range policies { + for _, st := range p.Statements { + if st.Effect != policy.Allow { + continue + } + probe := st + probe.Conditions = nil + if probe.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: action, + BucketName: bucket, + ObjectName: object, + ConditionValues: values, + Claims: cred.Claims, + }) { + setAllows = true + break + } + } + if setAllows { + break + } + } + if len(policies) > 0 && !setAllows { + return false + } + if len(policies) == 0 && cred.ParentUser != globalActiveCred.AccessKey && cred.AccessKey != globalActiveCred.AccessKey { + return false + } + } + return true +} + +func canReturnObjectTagsHeader(ctx context.Context, r *http.Request, versionID, existingTags string) bool { + reqInfo := logger.GetReqInfo(ctx) + if reqInfo == nil { + return false + } + prevVID := reqInfo.VersionID + reqInfo.VersionID = versionID + defer func() { reqInfo.VersionID = prevVID }() + + if isActionExplicitlyDenied(ctx, r, policy.GetObjectTaggingAction, existingTags, nil) { + return false + } + if verAction, ok := objectVersionAction(policy.GetObjectTaggingAction, versionID); ok { + if isActionExplicitlyDenied(ctx, r, verAction, existingTags, nil) { + return false + } + } + if authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectTaggingAction, versionID, existingTags) == ErrNone { + return true + } + return reqInfo.Cred.AccessKey != "" && + isRequestSignatureV4(r) && + !isRequestPresignedSignatureV4(r) && + authorizeRequestWithExistingTags(ctx, r, policy.ReplicateTagsAction, existingTags) == ErrNone +} + // authorizeRequestWithUnreadTags decides whether to disclose readErr for an // object whose stored tags could not be read. It must not grant object access. // @@ -53,20 +396,6 @@ func authorizeRequestWithUnreadTags(ctx context.Context, r *http.Request, action return authorizeRequestWithTagCandidates(ctx, r, action, s3Err) } -// authorizeRequestWithPotentialTags performs the authorization needed before -// a remote replica can be read. An empty tag set is enough to reject a caller -// with no possible access, but a caller whose Allow depends on existing tags -// must be admitted so the remote copy can supply the authoritative tags. -// The returned decision is only a gate; the actual tags must be authorized -// again after they are read. -func authorizeRequestWithPotentialTags(ctx context.Context, r *http.Request, action policy.Action) APIErrorCode { - s3Err := authorizeRequest(ctx, r, action) - if s3Err == ErrNone { - return ErrNone - } - return authorizeRequestWithTagCandidates(ctx, r, action, s3Err) -} - func authorizeRequestWithTagCandidates(ctx context.Context, r *http.Request, action policy.Action, s3Err APIErrorCode) APIErrorCode { for _, existingTags := range policyExistingObjectTags(ctx, r, action) { if authorizeRequestWithExistingTags(ctx, r, action, existingTags) == ErrNone { diff --git a/cmd/auth-handler.go b/cmd/auth-handler.go index b9af9bc53..0cde8e664 100644 --- a/cmd/auth-handler.go +++ b/cmd/auth-handler.go @@ -438,7 +438,12 @@ func deleteObjectAction(versionID string) policy.Action { func actionUsesObjectVersion(action policy.Action) bool { switch action { case policy.DeleteObjectAction, policy.DeleteObjectVersionAction, - policy.ReplicateDeleteAction, policy.BypassGovernanceRetentionAction: + policy.ReplicateDeleteAction, policy.BypassGovernanceRetentionAction, + policy.GetObjectAction, policy.GetObjectVersionAction, + policy.GetObjectAttributesAction, policy.GetObjectVersionAttributesAction, + policy.GetObjectTaggingAction, policy.GetObjectVersionTaggingAction, + policy.PutObjectTaggingAction, policy.PutObjectVersionTaggingAction, + policy.DeleteObjectTaggingAction, policy.DeleteObjectVersionTaggingAction: return true default: return false diff --git a/cmd/batch-rotate.go b/cmd/batch-rotate.go index d3d9c416d..b7188d45d 100644 --- a/cmd/batch-rotate.go +++ b/cmd/batch-rotate.go @@ -34,6 +34,8 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/minio/minio-go/v7/pkg/tags" "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/etag" + "github.com/minio/minio/internal/hash" xhttp "github.com/minio/minio/internal/http" "github.com/minio/minio/internal/kms" "github.com/pgsty/silo-pkg/v3/env" @@ -228,11 +230,11 @@ func (r *BatchJobKeyRotateV1) rotateAuthzRequests(oi ObjectInfo) []batchAuthzReq if versionID == "" { versionID = nullVersionID } - // The encryption the rewrite leaves: rotation keeps the kind of the - // version (rotateKey), and SSE-KMS takes the requested key. + // The encryption the rewrite leaves: SSE-KMS takes the requested key, + // including when migrating an SSE-S3 version to SSE-KMS. values := map[string][]string{} switch { - case crypto.S3KMS.IsEncrypted(oi.UserDefined): + case crypto.S3KMS.IsEncrypted(oi.UserDefined) || r.Encryption.Type == ssekms: values[strings.ToLower(xhttp.AmzServerSideEncryption)] = []string{xhttp.AmzEncryptionKMS} if r.Encryption.Type == ssekms && r.Encryption.Key != "" { values[strings.ToLower(xhttp.AmzServerSideEncryptionKmsID)] = []string{r.Encryption.Key} @@ -310,31 +312,113 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob } oi := obj.Clone() var ( - newKeyID string - newKeyContext kms.Context + newKeyID string + newKeyContext kms.Context + replaceContext bool + targetKind crypto.Type = crypto.S3 ) - encMetadata := make(map[string]string) - for k, v := range oi.UserDefined { - if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) { - encMetadata[k] = v - } + if sseKMS || r.Encryption.Type == ssekms { + targetKind = crypto.S3KMS } - if (sseKMS || sseS3) && r.Encryption.Type == ssekms { newKeyContext, err = r.Encryption.validate() if err != nil { return err } newKeyID = strings.TrimPrefix(r.Encryption.Key, crypto.ARNPrefix) - } - if err = rotateKey(ctx, []byte{}, newKeyID, []byte{}, r.Bucket, oi.Name, encMetadata, newKeyContext); err != nil { - return err + replaceContext = r.Encryption.Context != "" } - // Since we are rotating the keys, make sure to update the metadata. - oi.metadataOnly = true - oi.keyRotation = true - maps.Copy(oi.UserDefined, encMetadata) + srcOpts := ObjectOptions{VersionID: oi.VersionID} + dstOpts := ObjectOptions{ + VersionID: oi.VersionID, + Versioned: versioned, + VersionSuspended: versionSuspended, + NoLock: true, + } + + if oi.Legacy { + if targetKind == crypto.S3KMS && !replaceContext && len(newKeyContext) == 0 && sseKMS { + _, _, _, newKeyContext, err = crypto.S3KMS.ParseMetadata(oi.UserDefined) + if err != nil { + return err + } + } + readOpts := ObjectOptions{ + VersionID: oi.VersionID, + Versioned: versioned, + VersionSuspended: versionSuspended, + NoLock: true, + NoAuditLog: true, + } + var gr *GetObjectReader + if z, ok := api.(*erasureServerPools); ok && pool >= 0 { + gr, err = z.serverPools[pool].GetObjectNInfo(ctx, r.Bucket, encodeDirObject(oi.Name), nil, http.Header{}, readOpts) + } else { + gr, err = api.GetObjectNInfo(ctx, r.Bucket, oi.Name, nil, http.Header{}, readOpts) + } + if err != nil { + return err + } + defer gr.Close() + + actualSize, err := gr.ObjInfo.GetActualSize() + if err != nil { + return err + } + rawReader, err := hash.NewReader(ctx, gr, actualSize, "", "", actualSize) + if err != nil { + return err + } + pReader := NewPutObjReader(rawReader) + encMetadata := make(map[string]string) + encReader, objEncKey, err := newEncryptReader(ctx, rawReader, targetKind, newKeyID, nil, r.Bucket, oi.Name, encMetadata, newKeyContext) + if err != nil { + return err + } + crypto.RemoveInternalEntries(oi.UserDefined) + delete(oi.UserDefined, crypto.MetaContext) + delete(oi.UserDefined, ReservedMetadataPrefix+"compression") + delete(oi.UserDefined, ReservedMetadataPrefix+"actual-size") + maps.Copy(oi.UserDefined, encMetadata) + + targetSize := (&ObjectInfo{Size: actualSize}).EncryptedSize() + wrapped := etag.Wrap(encReader, rawReader) + hReader, err := hash.NewReader(ctx, wrapped, targetSize, "", "", actualSize) + if err != nil { + return err + } + pReader, err = pReader.WithEncryption(hReader, &objEncKey) + if err != nil { + return err + } + oi.PutObjReader = pReader + oi.metadataOnly = false + oi.keyRotation = true + dstOpts.EncryptFn = metadataEncrypter(objEncKey) + if dstOpts.VersionID == "" && (versioned || versionSuspended) { + dstOpts.VersionID = nullVersionID + } + } else { + encMetadata := make(map[string]string) + for k, v := range oi.UserDefined { + if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) { + encMetadata[k] = v + } + } + if err = rotateKeyWithTarget(ctx, []byte{}, targetKind, newKeyID, []byte{}, r.Bucket, oi.Name, encMetadata, newKeyContext, replaceContext); err != nil { + return err + } + // Since we are rotating the keys, make sure to update the metadata. + oi.metadataOnly = true + oi.keyRotation = true + delete(oi.UserDefined, crypto.MetaSealedKeyS3) + delete(oi.UserDefined, crypto.MetaSealedKeyKMS) + delete(oi.UserDefined, crypto.MetaKeyID) + delete(oi.UserDefined, crypto.MetaDataEncryptionKey) + delete(oi.UserDefined, crypto.MetaContext) + maps.Copy(oi.UserDefined, encMetadata) + } // The rewrite replaces the stored metadata with UserDefined, which a // read returns without the tags and the expiry: keep them. if oi.UserTags != "" { @@ -343,8 +427,6 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob if !oi.Expires.IsZero() { oi.UserDefined["expires"] = oi.ExpiresStr() } - srcOpts := ObjectOptions{VersionID: oi.VersionID} - dstOpts := ObjectOptions{VersionID: oi.VersionID, NoLock: true} if z, ok := api.(*erasureServerPools); ok && pool >= 0 { // Rewrite the copy that was read and authorized. The pools layer // would route the rewrite by the latest version of the key, which diff --git a/cmd/batch-submitter-authz_test.go b/cmd/batch-submitter-authz_test.go index bef964307..47233af10 100644 --- a/cmd/batch-submitter-authz_test.go +++ b/cmd/batch-submitter-authz_test.go @@ -33,6 +33,9 @@ func newBatchAuthzFixture(t *testing.T) *batchAuthzFixture { t.Helper() ctx, sys := iamAdminFixture(t) obj := newObjectLayerFn() + // Match storage-class parity to this fixture's single drive, even after + // an API test has loaded the configuration of a larger erasure set. + mustIAM(t, newTestConfig(globalMinioDefaultRegion, obj)) batchTestJobPool(ctx, t, obj) f := &batchAuthzFixture{ctx: ctx, sys: sys, obj: obj, bucket: "batch-authz"} mustIAM(t, obj.MakeBucket(ctx, f.bucket, MakeBucketOptions{VersioningEnabled: true})) diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index 07d1a06b3..b0bd94c8a 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -492,7 +492,14 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, deleteResults := make([]deleteResult, len(deleteObjectsReq.Objects)) duplicateOf := make(map[int]int) - vc, _ := globalBucketVersioningSys.Get(bucket) + vc, rcfg, metaErr := checkBucketDeleteMetadata(bucket) + if metaErr != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, metaErr), r.URL) + return + } + lockedNullDelete := func(obj ObjectToDelete) bool { + return obj.VersionID == "" && rcfg.LockEnabled && !vc.PrefixEnabled(obj.ObjectName) + } oss := make([]*objSweeper, len(deleteObjectsReq.Objects)) for index, object := range deleteObjectsReq.Objects { reqInfo.ObjectName = object.ObjectName @@ -536,7 +543,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, // Every physical version delete checks the version's own protection // under its mutation lock. Bucket metadata may be stale or unavailable // and cannot establish that a stored version is unprotected. - if object.VersionID != "" { + if object.VersionID != "" || lockedNullDelete(object) { if first, ok := objectsToDelete[object]; ok { duplicateOf[index] = first } else { @@ -601,23 +608,32 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, bulk := make([]ObjectToDelete, 0, len(deleteList)) bulkIndex := make([]int, 0, len(deleteList)) for i, object := range deleteList { - if object.VersionID == "" { + if object.VersionID == "" && !lockedNullDelete(object) { bulk = append(bulk, object) bulkIndex = append(bulkIndex, i) continue } reqInfo.ObjectName, reqInfo.VersionID = object.ObjectName, object.VersionID opts := ObjectOptions{ - VersionID: object.VersionID, - Versioned: vc.PrefixEnabled(object.ObjectName), - VersionSuspended: vc.Suspended(), + VersionID: object.VersionID, + Versioned: vc.PrefixEnabled(object.ObjectName), + VersionSuspended: vc.Suspended(), + RefuseLockedOverwrite: lockedNullDelete(object), } index := objectsToDelete[object] if !globalTierConfigMgr.Empty() { - oss[index] = newObjSweeper(bucket, object.ObjectName).WithVersion(object.VersionID).WithVersioning(opts.Versioned, opts.VersionSuspended) + sweepVID := object.VersionID + if sweepVID == "" && opts.VersionSuspended { + sweepVID = nullVersionID + } + oss[index] = newObjSweeper(bucket, object.ObjectName).WithVersion(sweepVID).WithVersioning(opts.Versioned, opts.VersionSuspended) } opts.SetEvalRetentionBypassFn(func(oi ObjectInfo, err error) error { - return enforceRetentionBypassForDelete(ctx, r, bucket, object, oi, err) + checkObj := object + if checkObj.VersionID == "" { + checkObj.VersionID = nullVersionID + } + return enforceRetentionBypassForDelete(ctx, r, bucket, checkObj, oi, err) }) opts.SetEvalMetadataFn(func(oi *ObjectInfo, err error) (ReplicateDecision, error) { if err == nil && oss[index] != nil { @@ -631,11 +647,17 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, // DeleteObject merges protection across pools before evaluating these // callbacks and holds the same lock as retention and legal-hold writes. oi, err := objectAPI.DeleteObject(ctx, bucket, object.ObjectName, opts) - dObjects[i] = DeletedObject{ + delObj := DeletedObject{ ObjectName: object.ObjectName, VersionID: object.VersionID, ReplicationState: oi.ReplicationState(), } + if object.VersionID == "" && oi.DeleteMarker { + delObj.DeleteMarker = true + delObj.DeleteMarkerVersionID = oi.VersionID + delObj.DeleteMarkerMTime = DeleteMarkerMTime{oi.ModTime} + } + dObjects[i] = delObj errs[i] = err } if len(bulk) > 0 { @@ -659,7 +681,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, // under the mutation lock, and a missing version has no stored identity. dindex := objectsToDelete[deleteList[i]] if errs[i] == nil || isErrObjectNotFound(errs[i]) || isErrVersionNotFound(errs[i]) { - if replicateDeletes && deleteList[i].VersionID == "" { + if replicateDeletes && deleteList[i].VersionID == "" && !lockedNullDelete(deleteList[i]) { dObjects[i].ReplicationState = deleteList[i].ReplicationState() } deleteResults[dindex].delInfo = dObjects[i] diff --git a/cmd/bucket-listobjects-handlers.go b/cmd/bucket-listobjects-handlers.go index 857b11087..d64f41eb1 100644 --- a/cmd/bucket-listobjects-handlers.go +++ b/cmd/bucket-listobjects-handlers.go @@ -90,8 +90,14 @@ func (api objectAPIHandlers) listObjectVersionsHandler(w http.ResponseWriter, r } var checkObjMeta metaCheckFn if metadata { - checkObjMeta = func(name string, action policy.Action) (s3Err APIErrorCode) { - return checkRequestAuthType(ctx, r, action, bucket, name) + checkObjMeta = func(obj ObjectInfo, action policy.Action) (s3Err APIErrorCode) { + vid := obj.VersionID + if vid == "" { + vid = nullVersionID + } + return withScopedReqObject(ctx, bucket, obj.Name, vid, func() APIErrorCode { + return authorizeVersionedRequestWithExistingTags(ctx, r, action, vid, obj.UserTags) + }) } } urlValues := r.Form @@ -176,8 +182,10 @@ func (api objectAPIHandlers) listObjectsV2Handler(ctx context.Context, w http.Re var checkObjMeta metaCheckFn if metadata { - checkObjMeta = func(name string, action policy.Action) (s3Err APIErrorCode) { - return checkRequestAuthType(ctx, r, action, bucket, name) + checkObjMeta = func(obj ObjectInfo, action policy.Action) (s3Err APIErrorCode) { + return withScopedReqObject(ctx, bucket, obj.Name, "", func() APIErrorCode { + return authorizeVersionedRequestWithExistingTags(ctx, r, action, "", obj.UserTags) + }) } } urlValues := r.Form diff --git a/cmd/bucket-metadata-replication.go b/cmd/bucket-metadata-replication.go index fdb181b46..58a9cd08e 100644 --- a/cmd/bucket-metadata-replication.go +++ b/cmd/bucket-metadata-replication.go @@ -182,8 +182,13 @@ func bucketConfigPayload(bucket, file string, data []byte, lockEnabled bool) ([] key := data switch file { case bucketPolicyConfig: + // Replication uses explicit null as a tombstone; ordinary policy PUTs + // validate their JSON before entering this metadata path. + if bytes.Equal(bytes.TrimSpace(data), []byte("null")) { + return nil, nil, nil + } var cfg *policy.BucketPolicy - cfg, err = policy.ParseBucketPolicyConfig(bytes.NewReader(data), bucket) + cfg, err = parseBucketPolicyStrictBytes(data, bucket) if err == nil { if cfg.IsEmpty() { return nil, nil, nil diff --git a/cmd/bucket-object-lock.go b/cmd/bucket-object-lock.go index d7c74d036..b21c2aa3c 100644 --- a/cmd/bucket-object-lock.go +++ b/cmd/bucket-object-lock.go @@ -28,11 +28,27 @@ import ( "github.com/minio/minio/internal/amztime" "github.com/minio/minio/internal/auth" objectlock "github.com/minio/minio/internal/bucket/object/lock" + "github.com/minio/minio/internal/bucket/versioning" xhttp "github.com/minio/minio/internal/http" "github.com/minio/minio/internal/logger" "github.com/pgsty/silo-pkg/v3/policy" ) +// checkBucketDeleteMetadata loads bucket versioning and Object Lock configs +// fail-closed so delete handlers never fall back to unversioned/unlocked +// defaults when bucket metadata cannot be read. +func checkBucketDeleteMetadata(bucketName string) (*versioning.Versioning, objectlock.Retention, error) { + vc, err := globalBucketVersioningSys.Get(bucketName) + if err != nil { + return nil, objectlock.Retention{}, err + } + rcfg, err := globalBucketObjectLockSys.Get(bucketName) + if err != nil { + return nil, objectlock.Retention{}, err + } + return vc, rcfg, nil +} + // BucketObjectLockSys - map of bucket and retention configuration. type BucketObjectLockSys struct{} diff --git a/cmd/bucket-policy-handlers.go b/cmd/bucket-policy-handlers.go index fe00f175b..264e0e000 100644 --- a/cmd/bucket-policy-handlers.go +++ b/cmd/bucket-policy-handlers.go @@ -18,7 +18,6 @@ package cmd import ( - "bytes" "io" "net/http" @@ -83,7 +82,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht return } - bucketPolicy, err := policy.ParseBucketPolicyConfig(bytes.NewReader(bucketPolicyBytes), bucket) + bucketPolicy, err := parseBucketPolicyStrictBytes(bucketPolicyBytes, bucket) if err != nil { writeErrorResponse(ctx, w, APIError{ Code: "MalformedPolicy", diff --git a/cmd/bucket-replication.go b/cmd/bucket-replication.go index fb942d2c0..a4d128a6c 100644 --- a/cmd/bucket-replication.go +++ b/cmd/bucket-replication.go @@ -2673,6 +2673,7 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, rs *HTTPRa }, PartNumber: opts.PartNumber, } + gopts.Set(xhttp.AmzTagDirective, accessDirective) if rs != nil { h, err := rs.ToHeader() if err != nil { diff --git a/cmd/encryption-v1.go b/cmd/encryption-v1.go index 32710915c..e7c7be4ae 100644 --- a/cmd/encryption-v1.go +++ b/cmd/encryption-v1.go @@ -260,6 +260,15 @@ func ParseSSECustomerHeader(header http.Header) (key []byte, err error) { // This function rotates old to new key. func rotateKey(ctx context.Context, oldKey []byte, newKeyID string, newKey []byte, bucket, object string, metadata map[string]string, cryptoCtx kms.Context) error { kind, _ := crypto.IsEncrypted(metadata) + return rotateKeyWithTarget(ctx, oldKey, kind, newKeyID, newKey, bucket, object, metadata, cryptoCtx, len(cryptoCtx) > 0) +} + +func rotateKeyWithTarget(ctx context.Context, oldKey []byte, targetKind crypto.Type, newKeyID string, newKey []byte, bucket, object string, metadata map[string]string, cryptoCtx kms.Context, replaceContext bool) error { + kind, _ := crypto.IsEncrypted(metadata) + if targetKind == nil { + targetKind = kind + } + var objectKey crypto.ObjectKey switch kind { case crypto.S3: if GlobalKMS == nil { @@ -269,7 +278,7 @@ func rotateKey(ctx context.Context, oldKey []byte, newKeyID string, newKey []byt if err != nil { return err } - oldKey, err := GlobalKMS.Decrypt(ctx, &kms.DecryptRequest{ + unsealKey, err := GlobalKMS.Decrypt(ctx, &kms.DecryptRequest{ Name: keyID, Ciphertext: kmsKey, AssociatedData: kms.Context{bucket: path.Join(bucket, object)}, @@ -277,66 +286,30 @@ func rotateKey(ctx context.Context, oldKey []byte, newKeyID string, newKey []byt if err != nil { return err } - var objectKey crypto.ObjectKey - if err = objectKey.Unseal(oldKey, sealedKey, crypto.S3.String(), bucket, object); err != nil { + if err = objectKey.Unseal(unsealKey, sealedKey, crypto.S3.String(), bucket, object); err != nil { return err } - - newKey, err := GlobalKMS.GenerateKey(ctx, &kms.GenerateKeyRequest{ - Name: GlobalKMS.DefaultKey, - AssociatedData: kms.Context{bucket: path.Join(bucket, object)}, - }) - if err != nil { - return err - } - sealedKey = objectKey.Seal(newKey.Plaintext, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object) - crypto.S3.CreateMetadata(metadata, newKey.KeyID, newKey.Ciphertext, sealedKey) - return nil case crypto.S3KMS: if GlobalKMS == nil { return errKMSNotConfigured } - objectKey, err := crypto.S3KMS.UnsealObjectKey(GlobalKMS, metadata, bucket, object) + var err error + objectKey, err = crypto.S3KMS.UnsealObjectKey(GlobalKMS, metadata, bucket, object) if err != nil { return err } - - if len(cryptoCtx) == 0 { + if targetKind == crypto.S3KMS && !replaceContext && len(cryptoCtx) == 0 { _, _, _, cryptoCtx, err = crypto.S3KMS.ParseMetadata(metadata) if err != nil { return err } } - - // If the context does not contain the bucket key - // we must add it for key generation. However, - // the context must be stored exactly like the - // client provided it. Therefore, we create a copy - // of the client provided context and add the bucket - // key, if not present. - kmsCtx := kms.Context{} - maps.Copy(kmsCtx, cryptoCtx) - if _, ok := kmsCtx[bucket]; !ok { - kmsCtx[bucket] = path.Join(bucket, object) - } - newKey, err := GlobalKMS.GenerateKey(ctx, &kms.GenerateKeyRequest{ - Name: newKeyID, - AssociatedData: kmsCtx, - }) - if err != nil { - return err - } - - sealedKey := objectKey.Seal(newKey.Plaintext, crypto.GenerateIV(rand.Reader), crypto.S3KMS.String(), bucket, object) - crypto.S3KMS.CreateMetadata(metadata, newKey.KeyID, newKey.Ciphertext, sealedKey, cryptoCtx) - return nil case crypto.SSEC: sealedKey, err := crypto.SSEC.ParseMetadata(metadata) if err != nil { return err } - var objectKey crypto.ObjectKey if err = objectKey.Unseal(oldKey, sealedKey, crypto.SSEC.String(), bucket, object); err != nil { if subtle.ConstantTimeCompare(oldKey, newKey) == 1 { return errInvalidSSEParameters // AWS returns special error for equal but invalid keys. @@ -353,6 +326,55 @@ func rotateKey(ctx context.Context, oldKey []byte, newKeyID string, newKey []byt default: return errObjectTampered } + + switch targetKind { + case crypto.S3: + genKey, err := GlobalKMS.GenerateKey(ctx, &kms.GenerateKeyRequest{ + Name: GlobalKMS.DefaultKey, + AssociatedData: kms.Context{bucket: path.Join(bucket, object)}, + }) + if err != nil { + return err + } + delete(metadata, crypto.MetaSealedKeyKMS) + delete(metadata, crypto.MetaKeyID) + delete(metadata, crypto.MetaDataEncryptionKey) + delete(metadata, crypto.MetaContext) + sealedKey := objectKey.Seal(genKey.Plaintext, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object) + crypto.S3.CreateMetadata(metadata, genKey.KeyID, genKey.Ciphertext, sealedKey) + return nil + case crypto.S3KMS: + // If the context does not contain the bucket key + // we must add it for key generation. However, + // the context must be stored exactly like the + // client provided it. Therefore, we create a copy + // of the client provided context and add the bucket + // key, if not present. + kmsCtx := kms.Context{} + maps.Copy(kmsCtx, cryptoCtx) + if _, ok := kmsCtx[bucket]; !ok { + kmsCtx[bucket] = path.Join(bucket, object) + } + genKey, err := GlobalKMS.GenerateKey(ctx, &kms.GenerateKeyRequest{ + Name: newKeyID, + AssociatedData: kmsCtx, + }) + if err != nil { + if errors.Is(err, kes.ErrKeyNotFound) { + return errKMSKeyNotFound + } + return err + } + delete(metadata, crypto.MetaSealedKeyS3) + delete(metadata, crypto.MetaKeyID) + delete(metadata, crypto.MetaDataEncryptionKey) + delete(metadata, crypto.MetaContext) + sealedKey := objectKey.Seal(genKey.Plaintext, crypto.GenerateIV(rand.Reader), crypto.S3KMS.String(), bucket, object) + crypto.S3KMS.CreateMetadata(metadata, genKey.KeyID, genKey.Ciphertext, sealedKey, cryptoCtx) + return nil + default: + return errInvalidEncryptionParameters + } } func newEncryptMetadata(ctx context.Context, kind crypto.Type, keyID string, key []byte, bucket, object string, metadata map[string]string, cryptoCtx kms.Context) (crypto.ObjectKey, error) { diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 53cde55f6..bbe73503c 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -963,11 +963,14 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s // to the disks; this value is not accurate because we do // not know the storage class of the object yet minDisks := 0 - if p := globalStorageClass.GetParityForSC(""); p > -1 { + if p := globalStorageClass.GetParityForSC(""); p > -1 && p < er.setDriveCount { minDisks = er.setDriveCount - p } else { minDisks = er.setDriveCount - er.defaultParityCount } + if minDisks < 1 { + minDisks = 1 + } if minDisks == er.setDriveCount/2 { // when data and parity are same we must atleast @@ -2217,7 +2220,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string // A restored-copy expiry needs fresh metadata to confirm its target // and deadline, a checked expiry to guard its protection; ordinary // unversioned DELETE may still try a blind delete. - if opts.Transition.ExpireRestored || checkedExpiry || opts.Versioned || opts.VersionSuspended || countOnlineDisks(storageDisks) < len(storageDisks)/2+1 { + if opts.Transition.ExpireRestored || checkedExpiry || opts.Versioned || opts.VersionSuspended || opts.RefuseLockedOverwrite || countOnlineDisks(storageDisks) < len(storageDisks)/2+1 { // Add an MRF heal for next time. er.addPartial(bucket, object, opts.VersionID) return objInfo, InsufficientWriteQuorum{} @@ -2268,8 +2271,28 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string } } + // When versioning is disabled or suspended for this key, an unaddressed + // delete overwrites or removes the null version even if a newer non-null + // version is returned as latest by getObjectInfoAndQuorum above. + retOI, retErr := goi, gerr + if !opts.Versioned && !opts.DeleteMarker && opts.VersionID == "" && (opts.EvalRetentionBypassFn != nil || opts.RefuseLockedOverwrite) { + if gerr == nil && goi.VersionID != "" && goi.VersionID != nullVersionID { + nullOpts := opts + nullOpts.VersionID = nullVersionID + retOI, _, retErr = er.getObjectInfoAndQuorum(ctx, bucket, object, nullOpts) + if _, ok := retErr.(InsufficientReadQuorum); ok && opts.RefuseLockedOverwrite { + er.addPartial(bucket, object, nullVersionID) + return objInfo, InsufficientWriteQuorum{} + } + } + } + if opts.EvalRetentionBypassFn != nil { - if err := opts.EvalRetentionBypassFn(goi, gerr); err != nil { + if err := opts.EvalRetentionBypassFn(retOI, retErr); err != nil { + return ObjectInfo{}, err + } + } else if opts.RefuseLockedOverwrite && !opts.Versioned && !opts.DeleteMarker && opts.VersionID == "" { + if err := refuseLockedMarkerOverwrite(ctx, bucket, object, nullVersionID, retOI, retErr); err != nil { return ObjectInfo{}, err } } diff --git a/cmd/erasure-server-pool-consistency.go b/cmd/erasure-server-pool-consistency.go index 71f99e3ee..d72e22e27 100644 --- a/cmd/erasure-server-pool-consistency.go +++ b/cmd/erasure-server-pool-consistency.go @@ -646,6 +646,26 @@ func (z *erasureServerPools) deleteObjectReconciled(ctx context.Context, bucket, case opts.VersionID != "": // An addressed version already resolved every copy above. logical = mergedPoolObjectInfo(copies) + case !opts.Versioned && !opts.DeleteMarker: + // An unqualified delete when versioning is not active for this key + // removes or replaces the null version in every pool that holds one, + // even if a newer non-null version is current for reads. + nullOpts := opts + nullOpts.VersionID = nullVersionID + nullCopies, err := z.objectPoolInfos(ctx, bucket, object, nullOpts) + switch { + case err == nil: + if nullCopies[0].ObjInfo.DeleteMarker { + gerr = toObjectErr(errFileNotFound, bucket, object) + logical = nullCopies[0].ObjInfo + } else { + logical = mergedPoolObjectInfo(nullCopies) + } + case isErrObjectNotFound(err) || isErrVersionNotFound(err): + gerr = toObjectErr(errFileNotFound, bucket, object) + default: + return ObjectInfo{}, err + } default: versions, err := z.metadataPoolInfos(ctx, bucket, object, opts) if err != nil { @@ -661,6 +681,14 @@ func (z *erasureServerPools) deleteObjectReconciled(ctx context.Context, bucket, return ObjectInfo{}, err } opts.EvalRetentionBypassFn = nil + if !opts.Versioned && !opts.DeleteMarker && opts.VersionID == "" { + opts.RefuseLockedOverwrite = false + } + } else if opts.RefuseLockedOverwrite && !opts.Versioned && !opts.DeleteMarker && opts.VersionID == "" { + if err := refuseLockedMarkerOverwrite(ctx, bucket, object, nullVersionID, logical, gerr); err != nil { + return ObjectInfo{}, err + } + opts.RefuseLockedOverwrite = false } if opts.EvalMetadataFn != nil { decision, err := opts.EvalMetadataFn(&logical, gerr) diff --git a/cmd/erasure-server-pool-multipart-lock-order_test.go b/cmd/erasure-server-pool-multipart-lock-order_test.go new file mode 100644 index 000000000..57a5526de --- /dev/null +++ b/cmd/erasure-server-pool-multipart-lock-order_test.go @@ -0,0 +1,205 @@ +// Copyright (c) 2026 PGSTY +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + xhttp "github.com/minio/minio/internal/http" +) + +type copyPartSourceContextKey struct{} + +// Pause the HTTP copy after the real reader has acquired its source lock. +// IAM and other background readers do not carry the request's context key. +type copyPartSourceGate struct { + ObjectLayer + opened, release chan struct{} +} + +func (a *copyPartSourceGate) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (*GetObjectReader, error) { + gr, err := a.ObjectLayer.GetObjectNInfo(ctx, bucket, object, rs, h, opts) + if err != nil || ctx.Value(copyPartSourceContextKey{}) != true { + return gr, err + } + close(a.opened) + select { + case <-a.release: + return gr, nil + case <-ctx.Done(): + gr.Close() + return nil, ctx.Err() + } +} + +func copyPartLockRefs(l *localLockInstance) int32 { + l.ns.lockMapMutex.Lock() + defer l.ns.lockMapMutex.Unlock() + if lock := l.ns.lockMap[pathJoin(l.volume, l.paths[0])]; lock != nil { + return lock.ref + } + return 0 +} + +// A copy from the upload's own key holds object RLock before asking for the +// upload RLock. Complete must wait for that reader without holding upload +// Lock, on both single- and multi-pool backends. Cancellation of either +// request must release its locks and let the other request finish. +func TestUploadPartCopyConcurrentComplete(t *testing.T) { + for _, pools := range []int{1, 2} { + for _, action := range []string{"complete", "cancel-copy", "cancel-complete"} { + t.Run(fmt.Sprintf("pools=%d/%s", pools, action), func(t *testing.T) { + z, _ := bulkDeletePools(t, pools) + ctx, cancel := context.WithTimeout(t.Context(), 15*time.Second) + var workers sync.WaitGroup + defer func() { cancel(); workers.Wait() }() + api := ©PartSourceGate{ObjectLayer: z, opened: make(chan struct{}), release: make(chan struct{})} + bucket, router, err := initAPIHandlerTest(ctx, api, nil, MakeBucketOptions{}) + if err != nil { + t.Fatal(err) + } + setObjectLayer(api) + const key = "copy-part-self" + data := bytes.Repeat([]byte("existing source bytes"), 32768) + oi, err := z.PutObject(ctx, bucket, key, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + if oi.Inlined { + t.Fatal("the source must retain its read lock until its reader closes") + } + upload, err := z.NewMultipartUpload(ctx, bucket, key, ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + first := []byte("the completed part") + part, err := z.PutObjectPart(ctx, bucket, key, upload.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(first), int64(len(first)), "", ""), ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + + objectLock := z.NewObjectNSLock(bucket, key).(*localLockInstance) + uploadLock := z.uploadIDNSLock(bucket, key, upload.UploadID).(*localLockInstance) + copyCtx, cancelCopy := context.WithCancel(context.WithValue(ctx, copyPartSourceContextKey{}, true)) + defer cancelCopy() + req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectPartURL("", bucket, key, upload.UploadID, "2"), 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, map[string]string{xhttp.AmzCopySource: "/" + bucket + "/" + key}) + if err != nil { + t.Fatal(err) + } + copyDone := make(chan *httptest.ResponseRecorder, 1) + workers.Go(func() { + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req.WithContext(copyCtx)) + copyDone <- rec + }) + select { + case <-api.opened: + case <-ctx.Done(): + t.Fatal("COPY did not open its source") + } + if refs := copyPartLockRefs(objectLock); refs != 1 { + t.Fatalf("COPY object lock references = %d, want 1", refs) + } + + completeCtx, cancelComplete := context.WithCancel(ctx) + defer cancelComplete() + completeDone := make(chan error, 1) + workers.Go(func() { + _, err := z.CompleteMultipartUpload(completeCtx, bucket, key, upload.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{}) + completeDone <- err + }) + // One reference belongs to COPY, the second to Complete waiting + // for it. Check the actual lock resources, not elapsed time. + ticker := time.NewTicker(time.Millisecond) + defer ticker.Stop() + for copyPartLockRefs(objectLock) != 2 { + select { + case err := <-completeDone: + t.Fatalf("Complete exited before waiting for COPY: %v", err) + case <-ctx.Done(): + t.Fatal("Complete did not wait for the source object lock") + case <-ticker.C: + } + } + if refs := copyPartLockRefs(uploadLock); refs != 0 { + t.Fatalf("Complete holds the upload lock while waiting for COPY's object lock: refs=%d", refs) + } + + if action == "cancel-complete" { + cancelComplete() + select { + case err := <-completeDone: + if !errors.Is(err, context.Canceled) { + t.Fatalf("canceled Complete: %v", err) + } + case <-ctx.Done(): + t.Fatal("Complete did not cancel while COPY held the source") + } + } + if action == "cancel-copy" { + cancelCopy() + } else { + close(api.release) + } + select { + case rec := <-copyDone: + want := http.StatusOK + if action == "cancel-copy" { + want = 499 + } + if rec.Code != want { + t.Fatalf("COPY: %d, want %d: %s", rec.Code, want, rec.Body.String()) + } + case <-ctx.Done(): + t.Fatal("COPY did not finish") + } + if action != "cancel-complete" { + select { + case err := <-completeDone: + if err != nil { + t.Fatalf("Complete after source release: %v", err) + } + case <-ctx.Done(): + t.Fatal("Complete did not finish after source release") + } + } + if copyPartLockRefs(objectLock) != 0 || copyPartLockRefs(uploadLock) != 0 { + t.Fatal("request left an object or upload lock behind") + } + wantBody := string(first) + if action == "cancel-complete" { + wantBody = string(data) + if _, err := z.GetMultipartInfo(ctx, bucket, key, upload.UploadID, ObjectOptions{}); err != nil { + t.Fatalf("canceled completion lost the upload: %v", err) + } + } + if body := bulkDeleteBody(t, z, bucket, key, ""); body != wantBody { + t.Fatalf("object body length = %d, want %d", len(body), len(wantBody)) + } + }) + } + } +} diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index 85f497abe..2ec9ba2f9 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -1320,7 +1320,9 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob approvedExpiry := opts.Expiration.Expire && opts.Expiration.CheckedAllPools && !opts.Transition.ExpireRestored reconcileVersion := opts.VersionID != "" && !opts.DataMovement && (!opts.ReplicationRequest || opts.isVersionPurge()) && (!opts.Expiration.Expire || approvedExpiry) && !opts.InclFreeVersions - if !z.SinglePool() && (opts.CheckPrecondFn != nil || reconcileVersion) { + reconcileUnaddressedLock := opts.VersionID == "" && !opts.Versioned && !opts.DeleteMarker && + opts.RefuseLockedOverwrite && !opts.DataMovement && !opts.ReplicationRequest && !opts.Expiration.Expire && !opts.InclFreeVersions + if !z.SinglePool() && (opts.CheckPrecondFn != nil || reconcileVersion || reconcileUnaddressedLock) { return z.deleteObjectReconciled(ctx, bucket, object, opts) } @@ -1822,6 +1824,10 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec } } + if srcInfo.PutObjReader == nil { + return ObjectInfo{}, errInvalidArgument + } + putOpts := ObjectOptions{ ServerSideEncryption: dstOpts.ServerSideEncryption, UserDefined: srcInfo.UserDefined, @@ -1834,6 +1840,7 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec IndexCB: dstOpts.IndexCB, WantChecksum: dstOpts.WantChecksum, WantServerSideChecksumType: dstOpts.WantServerSideChecksumType, + PreserveETag: dstOpts.PreserveETag, ReplicationRequest: dstOpts.ReplicationRequest, ReplicaLockReconcile: dstOpts.ReplicaLockReconcile, DataMovement: dstOpts.DataMovement, @@ -2499,21 +2506,21 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket } }() - // Several pools take the object lock first, then the upload lock below. - // A single pool takes the upload lock here and the set takes the object - // lock, in the object's own erasure set, only right before it commits. - if !z.SinglePool() { - if !opts.NoLock { - lk := z.NewObjectNSLock(bucket, encodeDirObject(object)) - lkctx, err := lk.GetLock(ctx, globalOperationTimeout) - if err != nil { - return objInfo, err - } - ctx = lkctx.Context() - defer lk.Unlock(lkctx) + // Take the object lock before the upload lock for every pool count. + // UploadPartCopy holds its source read lock while it reads upload + // metadata, including when the source and destination are the same key. + if !opts.NoLock { + lk := z.NewObjectNSLock(bucket, encodeDirObject(object)) + lkctx, err := lk.GetLock(ctx, globalOperationTimeout) + if err != nil { + return objInfo, err } - opts.NoLock = true + ctx = lkctx.Context() + defer lk.Unlock(lkctx) + } + opts.NoLock = true + if !z.SinglePool() { // A conditional completion must be evaluated against the logical // latest object across pools, under the object write lock held for // this operation. The pool hosting the upload may only hold a stale diff --git a/cmd/erasure-sets.go b/cmd/erasure-sets.go index b20a1ce90..5dfc5b38c 100644 --- a/cmd/erasure-sets.go +++ b/cmd/erasure-sets.go @@ -844,7 +844,7 @@ func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstB // Check if this request is only metadata update. // CopyObjectHandler predicts the outcome of this decision in // copyRewritesObjectData(); keep the two in sync. - if cpSrcDstSame && srcInfo.metadataOnly { + if cpSrcDstSame && srcInfo.metadataOnly && !srcInfo.Legacy { // Version ID is set for the destination and source == destination version ID. // perform an in-place update. if dstOpts.VersionID != "" && srcOpts.VersionID == dstOpts.VersionID { @@ -868,6 +868,10 @@ func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstB } } + if srcInfo.PutObjReader == nil { + return ObjectInfo{}, errInvalidArgument + } + putOpts := ObjectOptions{ ServerSideEncryption: dstOpts.ServerSideEncryption, UserDefined: srcInfo.UserDefined, @@ -879,10 +883,12 @@ func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstB EncryptFn: dstOpts.EncryptFn, WantChecksum: dstOpts.WantChecksum, WantServerSideChecksumType: dstOpts.WantServerSideChecksumType, + PreserveETag: dstOpts.PreserveETag, ReplicationRequest: dstOpts.ReplicationRequest, ReplicaLockReconcile: dstOpts.ReplicaLockReconcile, DataMovement: dstOpts.DataMovement, replicaObjectInfo: dstOpts.replicaObjectInfo, + replicaContentGuard: dstOpts.replicaContentGuard, } return dstSet.putObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, putOpts) diff --git a/cmd/iam-replication-protocol.go b/cmd/iam-replication-protocol.go index fe1814823..be20062e0 100644 --- a/cmd/iam-replication-protocol.go +++ b/cmd/iam-replication-protocol.go @@ -4,7 +4,6 @@ package cmd import ( - "bytes" "context" "crypto/sha256" "encoding/hex" @@ -302,7 +301,7 @@ func applyIAMReplicationItem(ctx context.Context, item iamReplicationItem) error if len(item.Policy) == 0 { return globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt) } - p, err := policy.ParseConfig(bytes.NewReader(item.Policy)) + p, err := parsePolicySnapshotBytes(item.Policy) if err != nil { return err } diff --git a/cmd/iam.go b/cmd/iam.go index aa115c0a1..21cfc4a7f 100644 --- a/cmd/iam.go +++ b/cmd/iam.go @@ -1087,17 +1087,26 @@ func (sys *IAMSys) notifyForServiceAccount(ctx context.Context, accessKey string } type newServiceAccountOpts struct { - sessionPolicy *policy.Policy - accessKey string - secretKey string - status string // Used by replication snapshots; local creates default to enabled. - name, description string - expiration *time.Time - allowSiteReplicatorAccount bool // allow creating internal service account for site-replication. + sessionPolicy *policy.Policy + accessKey string + secretKey string + status string // Used by replication snapshots; local creates default to enabled. + name, description string + expiration *time.Time + allowSiteReplicatorAccount bool // allow creating internal service account for site-replication. + preserveEmptyEmbeddedPolicy bool // preserve historical deny-all empty embedded policies on snapshot import/replication. claims map[string]any } +func svcAcctClaimsHaveEmbeddedPolicy(claims map[string]any) bool { + if claims == nil { + return false + } + pt, _ := claims[iamPolicyClaimNameSA()].(string) + return pt == embeddedPolicyType +} + // marshalServiceAccountPolicy validates a supplied policy and normalizes its // version without changing the caller's policy. An empty policy means inheritance. func marshalServiceAccountPolicy(sp *policy.Policy) ([]byte, error) { @@ -1157,10 +1166,32 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro m := make(map[string]any) m[parentClaim] = parentUser - if len(policyBuf) > 0 { + switch { + case len(policyBuf) > 0: m[policy.SessionPolicyName] = base64.StdEncoding.EncodeToString(policyBuf) m[iamPolicyClaimNameSA()] = embeddedPolicyType - } else { + case opts.preserveEmptyEmbeddedPolicy: + switch { + case opts.sessionPolicy != nil: + p := *opts.sessionPolicy + if p.Version == "" { + p.Version = policy.DefaultVersion + } + buf, merr := json.Marshal(p) + if merr != nil { + return auth.Credentials{}, time.Time{}, merr + } + m[policy.SessionPolicyName] = base64.StdEncoding.EncodeToString(buf) + m[iamPolicyClaimNameSA()] = embeddedPolicyType + default: + if raw, ok := opts.claims[policy.SessionPolicyName].(string); ok && raw != "" { + m[policy.SessionPolicyName] = raw + m[iamPolicyClaimNameSA()] = embeddedPolicyType + } else { + m[iamPolicyClaimNameSA()] = inheritedPolicyType + } + } + default: m[iamPolicyClaimNameSA()] = inheritedPolicyType } diff --git a/cmd/object-api-options.go b/cmd/object-api-options.go index 9aeee35fd..802d7b227 100644 --- a/cmd/object-api-options.go +++ b/cmd/object-api-options.go @@ -303,13 +303,18 @@ func delOpts(ctx context.Context, r *http.Request, bucket, object string) (opts } } + vc, _, err := checkBucketDeleteMetadata(bucket) + if err != nil { + return opts, err + } + opts.DeletePrefix = deletePrefix - opts.Versioned = globalBucketVersioningSys.PrefixEnabled(bucket, object) + opts.Versioned = vc.PrefixEnabled(object) // Objects matching prefixes should not leave delete markers, // dramatically reduces namespace pollution while keeping the // benefits of replication, make sure to apply version suspension // only at bucket level instead. - opts.VersionSuspended = globalBucketVersioningSys.Suspended(bucket) + opts.VersionSuspended = vc.Suspended() // For directory objects, delete `null` version permanently. if isDirObject(object) && opts.VersionID == "" { opts.VersionID = nullVersionID diff --git a/cmd/object-copy-replication-tagging_test.go b/cmd/object-copy-replication-tagging_test.go index e187663d6..2e1ba87cc 100644 --- a/cmd/object-copy-replication-tagging_test.go +++ b/cmd/object-copy-replication-tagging_test.go @@ -35,22 +35,15 @@ import ( // the single-disk and 16-disk fixtures are single-pool backends. func TestAPICopyObjectReplicaTaggingTimestampUnderKMS(t *testing.T) { defer DetectTestLeak(t)() - ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPICopyObjectReplicaTaggingTimestampUnderKMS}) + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, objAPITest: testAPICopyObjectReplicaTaggingTimestampUnderKMS, + // Ignore host free-space percentage while retaining real disk I/O. + // Install once before IAM starts and keep the disks until shutdown. + wrapDisk: func(disk StorageAPI) StorageAPI { return tagTestCapacityDisk{StorageAPI: disk} }, + }) } func testAPICopyObjectReplicaTaggingTimestampUnderKMS(obj ObjectLayer, instance, bucket string, router http.Handler, creds auth.Credentials, t *testing.T) { - // Ignore the host free-space percentage while retaining real disk I/O. - for _, pool := range obj.(*erasureServerPools).serverPools { - for _, set := range pool.sets { - original := set.getDisks - disks := append([]StorageAPI(nil), original()...) - for i := range disks { - disks[i] = tagTestCapacityDisk{StorageAPI: disks[i]} - } - set.getDisks = func() []StorageAPI { return disks } - defer func() { set.getDisks = original }() - } - } oldKMS, oldAuto := GlobalKMS, globalAutoEncryption GlobalKMS = kms.NewStub("replica-tags-key") globalAutoEncryption = false diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 0a2b2412a..db59259bf 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -139,7 +139,13 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r // Check for auth type to return S3 compatible error. // type to return the correct error (NoSuchKey vs AccessDenied) - if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, bucket, object); s3Error != ErrNone { + logger.GetReqInfo(ctx).BucketName = bucket + logger.GetReqInfo(ctx).ObjectName = object + s3Error := authenticateRequest(ctx, r, policy.GetObjectAction) + if s3Error == ErrNone { + s3Error = authorizeVersionedRequestWithPotentialTags(ctx, r, policy.GetObjectAction, opts.VersionID) + } + if s3Error != ErrNone { if getRequestAuthType(r) == authTypeAnonymous { // As per "Permission" section in // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html @@ -199,6 +205,19 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r gopts.NoLock = true // We already have a lock, we can live with it. objInfo, err := getObjectInfo(ctx, bucket, object, gopts) if err != nil { + if s3Error := authorizeVersionedRequestWithUnreadTags(ctx, r, policy.GetObjectAction, opts.VersionID, err); s3Error != ErrNone { + if getRequestAuthType(r) == authTypeAnonymous && toAPIError(ctx, err).Code == "NoSuchKey" && + globalPolicySys.IsAllowed(policy.BucketPolicyArgs{ + Action: policy.ListBucketAction, + BucketName: bucket, + ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials), + IsOwner: false, + }) { + s3Error = ErrNoSuchKey + } + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } // Versioning enabled quite possibly object is deleted might be delete-marker // if present set the headers, no idea why AWS S3 sets these headers. if objInfo.VersionID != "" && objInfo.DeleteMarker { @@ -208,10 +227,14 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return } + if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, opts.VersionID, objInfo.UserTags); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } // filter object lock metadata if permission does not permit - getRetPerms := checkRequestAuthType(ctx, r, policy.GetObjectRetentionAction, bucket, object) - legalHoldPerms := checkRequestAuthType(ctx, r, policy.GetObjectLegalHoldAction, bucket, object) + getRetPerms := authorizeRequest(ctx, r, policy.GetObjectRetentionAction) + legalHoldPerms := authorizeRequest(ctx, r, policy.GetObjectLegalHoldAction) // filter object lock metadata if permission does not permit objInfo.UserDefined = objectlock.FilterObjectLockMetadata(objInfo.UserDefined, getRetPerms != ErrNone, legalHoldPerms != ErrNone) @@ -326,6 +349,8 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj // Check for auth type to return S3 compatible error. // type to return the correct error (NoSuchKey vs AccessDenied) + logger.GetReqInfo(ctx).BucketName = bucket + logger.GetReqInfo(ctx).ObjectName = object if s3Error := authenticateRequest(ctx, r, policy.GetObjectAction); s3Error != ErrNone { if getRequestAuthType(r) == authTypeAnonymous { // As per "Permission" section in @@ -397,14 +422,14 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj // Validate pre-conditions if any. var storedTags *string // set once the object has been read far enough opts.CheckPrecondFn = func(oi ObjectInfo) bool { - if _, err := DecryptObjectInfo(&oi, r); err != nil { - writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + storedTags = &oi.UserTags + if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, opts.VersionID, oi.UserTags); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) return true } - storedTags = &oi.UserTags - if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.GetObjectAction, oi.UserTags); s3Error != ErrNone { - writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + if _, err := DecryptObjectInfo(&oi, r); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return true } @@ -423,7 +448,7 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj if (isErrObjectNotFound(err) || isErrVersionNotFound(err) || isErrReadQuorum(err)) && (gr == nil || !gr.ObjInfo.DeleteMarker) { proxytgts := getProxyTargets(ctx, bucket, object, opts) - if !proxytgts.Empty() { + if !proxytgts.Empty() && authorizeVersionedRequestWithPotentialTags(ctx, r, policy.GetObjectAction, opts.VersionID) == ErrNone { globalReplicationStats.Load().incProxy(bucket, getObjectAPI, false) // proxy to replication target if active-active replication is in place. reader, proxy, perr = proxyGetToReplicationTarget(ctx, bucket, object, rs, r.Header, opts, proxytgts) @@ -441,18 +466,27 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj } } if reader == nil || !proxy.Proxy { - // The conditional callback has already written 304/412. Do not + // The conditional callback has already written 304/412 (or 403). Do not // authorize again without the stored tags or write a second response. - if isErrPreconditionFailed(err) { + if isErrPreconditionFailed(err) || isErrPreconditionFailed(perr) { return } // validate if the request indeed was authorized, if it wasn't we need to return "ErrAccessDenied" // instead of any namespace related error. Use the stored tags if the callback read them. var s3Error APIErrorCode if storedTags != nil { - s3Error = authorizeRequestWithExistingTags(ctx, r, policy.GetObjectAction, *storedTags) + s3Error = authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, opts.VersionID, *storedTags) } else { - s3Error = authorizeRequestWithUnreadTags(ctx, r, policy.GetObjectAction, err) + readErr := err + if proxy.Err != nil { + readErr = proxy.Err + } + if getRequestAuthType(r) == authTypeAnonymous && toAPIError(ctx, readErr).Code == "NoSuchKey" && + authorizeBucketRequest(ctx, r, policy.ListBucketAction) == ErrNone { + s3Error = ErrNone + } else { + s3Error = authorizeVersionedRequestWithUnreadTags(ctx, r, policy.GetObjectAction, opts.VersionID, readErr) + } } if s3Error != ErrNone { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) @@ -486,6 +520,12 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj defer gr.Close() objInfo := gr.ObjInfo + if storedTags == nil { + if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, opts.VersionID, objInfo.UserTags); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + } if !proxy.Proxy { // apply lifecycle rules only for local requests // Automatically remove the object/version if an expiry lifecycle rule can be applied @@ -547,6 +587,10 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj hash.AddChecksumHeader(w, cs) } + if opts.Tagging && !canReturnObjectTagsHeader(ctx, r, opts.VersionID, objInfo.UserTags) { + opts.Tagging = false + } + if err = setObjectHeaders(ctx, w, objInfo, rs, opts); err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return @@ -611,6 +655,7 @@ func (api objectAPIHandlers) getObjectAttributesHandler(ctx context.Context, obj } logger.GetReqInfo(ctx).BucketName = bucket logger.GetReqInfo(ctx).ObjectName = object + logger.GetReqInfo(ctx).VersionID = opts.VersionID if s3Error := authenticateRequest(ctx, r, actions[0]); s3Error != ErrNone { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) return @@ -851,6 +896,8 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob // Check for auth type to return S3 compatible error. // type to return the correct error (NoSuchKey vs AccessDenied) + logger.GetReqInfo(ctx).BucketName = bucket + logger.GetReqInfo(ctx).ObjectName = object if s3Error := authenticateRequest(ctx, r, policy.GetObjectAction); s3Error != ErrNone { if getRequestAuthType(r) == authTypeAnonymous { // As per "Permission" section in @@ -928,32 +975,43 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob if err != nil && !objInfo.DeleteMarker && (isErrObjectNotFound(err) || isErrVersionNotFound(err) || isErrReadQuorum(err)) { // proxy HEAD to replication target if active-active replication configured on bucket proxytgts := getProxyTargets(ctx, bucket, object, opts) - if !proxytgts.Empty() { + if !proxytgts.Empty() && authorizeVersionedRequestWithPotentialTags(ctx, r, policy.GetObjectAction, opts.VersionID) == ErrNone { globalReplicationStats.Load().incProxy(bucket, headObjectAPI, false) var oi ObjectInfo oi, proxy = proxyHeadToReplicationTarget(ctx, bucket, object, rs, opts, proxytgts) - if proxy.Proxy { + if proxy.Proxy && proxy.Err == nil { objInfo = oi } if proxy.Err != nil { globalReplicationStats.Load().incProxy(bucket, headObjectAPI, true) - writeErrorResponseHeadersOnly(w, toAPIError(ctx, proxy.Err)) - return } } } // objInfo carries the stored tags unless reading the object failed. var s3Error APIErrorCode - if err != nil && !proxy.Proxy { - s3Error = authorizeRequestWithUnreadTags(ctx, r, policy.GetObjectAction, err) + if (err != nil && !proxy.Proxy) || proxy.Err != nil { + readErr := err + if proxy.Err != nil { + readErr = proxy.Err + } + if getRequestAuthType(r) == authTypeAnonymous && toAPIError(ctx, readErr).Code == "NoSuchKey" && + authorizeBucketRequest(ctx, r, policy.ListBucketAction) == ErrNone { + s3Error = ErrNone + } else { + s3Error = authorizeVersionedRequestWithUnreadTags(ctx, r, policy.GetObjectAction, opts.VersionID, readErr) + } } else { - s3Error = authorizeRequestWithExistingTags(ctx, r, policy.GetObjectAction, objInfo.UserTags) + s3Error = authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, opts.VersionID, objInfo.UserTags) } if s3Error != ErrNone { writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(s3Error)) return } + if proxy.Err != nil { + writeErrorResponseHeadersOnly(w, toAPIError(ctx, proxy.Err)) + return + } if err != nil && !proxy.Proxy { switch { @@ -1058,6 +1116,10 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob hash.AddChecksumHeader(w, cs) } + if opts.Tagging && !canReturnObjectTagsHeader(ctx, r, opts.VersionID, objInfo.UserTags) { + opts.Tagging = false + } + // Set standard object headers. if err = setObjectHeaders(ctx, w, objInfo, rs, opts); err != nil { writeErrorResponseHeadersOnly(w, toAPIError(ctx, err)) @@ -1366,9 +1428,23 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re return } - if s3Error := checkRequestAuthType(ctx, r, policy.PutObjectAction, dstBucket, dstObject); s3Error != ErrNone { - writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) - return + if isDirectiveReplace(r.Header.Get(xhttp.AmzTagDirective)) { + if s3Error := checkRequestAuthType(ctx, r, policy.PutObjectAction, dstBucket, dstObject); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + } else { + logger.GetReqInfo(ctx).BucketName = dstBucket + logger.GetReqInfo(ctx).ObjectName = dstObject + logger.GetReqInfo(ctx).VersionID = "" + if s3Error := authenticateRequest(ctx, r, policy.PutObjectAction); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + if !canPotentiallyAllowAction(ctx, r, policy.PutObjectAction, dstBucket, dstObject) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } } // Read escaped copy source path to check for parameters. @@ -1402,7 +1478,9 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re } } - if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, srcBucket, srcObject); s3Error != ErrNone { + if s3Error := withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode { + return authorizeVersionedRequestWithPotentialTags(ctx, r, policy.GetObjectAction, vid) + }); s3Error != ErrNone { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) return } @@ -1486,7 +1564,15 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re getObjectNInfo := objectAPI.GetObjectNInfo + var storedSrcTags *string checkCopyPrecondFn := func(o ObjectInfo) bool { + storedSrcTags = &o.UserTags + if s3Error := withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode { + return authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, vid, o.UserTags) + }); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return true + } if _, err := DecryptObjectInfo(&o, r); err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return true @@ -1538,6 +1624,20 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re if isErrPreconditionFailed(err) { return } + var s3Error APIErrorCode + if storedSrcTags != nil { + s3Error = withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode { + return authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, vid, *storedSrcTags) + }) + } else { + s3Error = withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode { + return authorizeVersionedRequestWithUnreadTags(ctx, r, policy.GetObjectAction, vid, err) + }) + } + if s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } // Versioning enabled quite possibly object is deleted might be delete-marker // if present set the headers, no idea why AWS S3 sets these headers. @@ -1555,6 +1655,23 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re } defer gr.Close() srcInfo := gr.ObjInfo + if storedSrcTags == nil { + if s3Error := withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode { + return authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, vid, srcInfo.UserTags) + }); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + } + if !isDirectiveReplace(r.Header.Get(xhttp.AmzTagDirective)) { + copiedTags := srcInfo.UserTags + if s3Error := withScopedReqObject(ctx, dstBucket, dstObject, "", func() APIErrorCode { + return authorizeRequestWithTags(ctx, r, policy.PutObjectAction, "", &copiedTags) + }); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + } // A trusted SSE-C replica reads raw ciphertext. Federation's ordinary PUT // expects plaintext and does not forward the replica's sealed-key metadata. if remoteCallRequired && replicaTrusted && crypto.SSEC.IsEncrypted(srcInfo.UserDefined) { @@ -1931,6 +2048,17 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re srcInfo.UserDefined[xhttp.AmzObjectTagging] = objTags srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = monotonicMetadataTimestamp(UTCNow().Format(time.RFC3339Nano), localTagTimestamp) } + if !isDirectiveReplace(r.Header.Get(xhttp.AmzTagDirective)) { + finalDstTags := srcInfo.UserDefined[xhttp.AmzObjectTagging] + if finalDstTags != srcInfo.UserTags { + if s3Error := withScopedReqObject(ctx, dstBucket, dstObject, "", func() APIErrorCode { + return authorizeRequestWithTags(ctx, r, policy.PutObjectAction, "", &finalDstTags) + }); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + } + } // SSE-C rotation snapshots reserved metadata before the tag decision. Its // later merge must not put the old timestamp back over the accepted state. delete(encMetadata, ReservedMetadataPrefixLower+TaggingTimestamp) @@ -2293,9 +2421,13 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return } + if size < 0 { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidArgument), r.URL) + return + } } } - if size == -1 { + if size < 0 { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL) return } @@ -2758,10 +2890,14 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return } + if size < 0 { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidArgument), r.URL) + return + } } } - if size == -1 { + if size < 0 { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL) return } @@ -3255,6 +3391,12 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http. return dsc, nil }) + _, rcfg, err := checkBucketDeleteMetadata(bucket) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + vID := opts.VersionID if replica { opts.SetReplicaStatus(replication.Replica) @@ -3266,20 +3408,27 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http. opts.RefuseLockedOverwrite = true } } + checkUnaddressedLock := vID == "" && !opts.Versioned && !opts.DeleteMarker && rcfg.LockEnabled + if checkUnaddressedLock { + opts.RefuseLockedOverwrite = true + } opts.SetEvalRetentionBypassFn(func(goi ObjectInfo, gerr error) (err error) { - err = nil - if vID != "" { + if vID != "" || checkUnaddressedLock || (vID == "" && !opts.Versioned && !opts.DeleteMarker && gerr == nil) { + checkVID := vID + if checkVID == "" { + checkVID = nullVersionID + } err := enforceRetentionBypassForDelete(ctx, r, bucket, ObjectToDelete{ ObjectV: ObjectV{ ObjectName: object, - VersionID: vID, + VersionID: checkVID, }, }, goi, gerr) if err != nil && !isErrObjectNotFound(err) { return err } } - return err + return nil }) deleteObject := objectAPI.DeleteObject @@ -3733,6 +3882,8 @@ func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *h return } + logger.GetReqInfo(ctx).BucketName = bucket + logger.GetReqInfo(ctx).ObjectName = object if s3Error := authenticateRequest(ctx, r, policy.GetObjectTaggingAction); s3Error != ErrNone { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) return @@ -3752,7 +3903,7 @@ func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *h if isErrObjectNotFound(err) || isErrVersionNotFound(err) { proxytgts := getProxyTargets(ctx, bucket, object, opts) if !proxytgts.Empty() { - if s3Error := authorizeRequestWithPotentialTags(ctx, r, policy.GetObjectTaggingAction); s3Error != ErrNone { + if s3Error := authorizeVersionedRequestWithPotentialTags(ctx, r, policy.GetObjectTaggingAction, opts.VersionID); s3Error != ErrNone { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) return } @@ -3760,9 +3911,9 @@ func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *h // proxy to replication target if site replication is in place. tags, gerr, s3Error := proxyGetTaggingToRepTarget(ctx, bucket, object, opts, proxytgts, func(existingTags string, readErr error) APIErrorCode { if readErr != nil && !isRemoteTaggingNotFound(readErr) { - return authorizeRequestWithUnreadTags(ctx, r, policy.GetObjectTaggingAction, readErr) + return authorizeVersionedRequestWithUnreadTags(ctx, r, policy.GetObjectTaggingAction, opts.VersionID, readErr) } - return authorizeRequestWithExistingTags(ctx, r, policy.GetObjectTaggingAction, existingTags) + return authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectTaggingAction, opts.VersionID, existingTags) }) if s3Error != ErrNone { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) @@ -3776,7 +3927,7 @@ func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *h ot = tags w.Header()[xhttp.MinIOTaggingProxied] = []string{"true"} // indicate that the request was proxied. } else { - if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.GetObjectTaggingAction, ""); s3Error != ErrNone { + if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectTaggingAction, opts.VersionID, ""); s3Error != ErrNone { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) return } @@ -3784,7 +3935,7 @@ func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *h return } } else { - if s3Error := authorizeRequestWithUnreadTags(ctx, r, policy.GetObjectTaggingAction, err); s3Error != ErrNone { + if s3Error := authorizeVersionedRequestWithUnreadTags(ctx, r, policy.GetObjectTaggingAction, opts.VersionID, err); s3Error != ErrNone { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) return } @@ -3793,7 +3944,7 @@ func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *h } } - if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.GetObjectTaggingAction, ot.String()); s3Error != ErrNone { + if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectTaggingAction, opts.VersionID, ot.String()); s3Error != ErrNone { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) return } @@ -3888,15 +4039,15 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h if isErrObjectNotFound(err) || isErrVersionNotFound(err) { proxytgts := getProxyTargets(ctx, bucket, object, opts) if !proxytgts.Empty() { - if s3Error := authorizeRequestWithPotentialTags(ctx, r, policy.PutObjectTaggingAction); s3Error != ErrNone { + if s3Error := authorizeVersionedRequestWithPotentialTags(ctx, r, policy.PutObjectTaggingAction, opts.VersionID); s3Error != ErrNone { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) return } perr, s3Error := proxyTaggingToRepTarget(ctx, bucket, object, tags, opts, proxytgts, func(existingTags string, readErr error) APIErrorCode { if readErr != nil && !isRemoteTaggingNotFound(readErr) { - return authorizeRequestWithUnreadTags(ctx, r, policy.PutObjectTaggingAction, readErr) + return authorizeVersionedRequestWithUnreadTags(ctx, r, policy.PutObjectTaggingAction, opts.VersionID, readErr) } - return authorizeRequestWithExistingTags(ctx, r, policy.PutObjectTaggingAction, existingTags) + return authorizeVersionedRequestWithExistingTags(ctx, r, policy.PutObjectTaggingAction, opts.VersionID, existingTags) }) if s3Error != ErrNone { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) @@ -3928,9 +4079,9 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h // caller for retry, while a missing object is evaluated as untagged. var s3Error APIErrorCode if isErrObjectNotFound(err) || isErrVersionNotFound(err) { - s3Error = authorizeRequestWithExistingTags(ctx, r, policy.PutObjectTaggingAction, "") + s3Error = authorizeVersionedRequestWithExistingTags(ctx, r, policy.PutObjectTaggingAction, opts.VersionID, "") } else { - s3Error = authorizeRequestWithUnreadTags(ctx, r, policy.PutObjectTaggingAction, err) + s3Error = authorizeVersionedRequestWithUnreadTags(ctx, r, policy.PutObjectTaggingAction, opts.VersionID, err) } if s3Error != ErrNone { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) @@ -3939,7 +4090,7 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return } - if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.PutObjectTaggingAction, objInfo.UserTags); s3Error != ErrNone { + if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.PutObjectTaggingAction, opts.VersionID, objInfo.UserTags); s3Error != ErrNone { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) return } @@ -4020,15 +4171,15 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r if isErrObjectNotFound(err) || isErrVersionNotFound(err) { proxytgts := getProxyTargets(ctx, bucket, object, opts) if !proxytgts.Empty() { - if s3Error := authorizeRequestWithPotentialTags(ctx, r, policy.DeleteObjectTaggingAction); s3Error != ErrNone { + if s3Error := authorizeVersionedRequestWithPotentialTags(ctx, r, policy.DeleteObjectTaggingAction, opts.VersionID); s3Error != ErrNone { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) return } perr, s3Error := proxyTaggingToRepTarget(ctx, bucket, object, nil, opts, proxytgts, func(existingTags string, readErr error) APIErrorCode { if readErr != nil && !isRemoteTaggingNotFound(readErr) { - return authorizeRequestWithUnreadTags(ctx, r, policy.DeleteObjectTaggingAction, readErr) + return authorizeVersionedRequestWithUnreadTags(ctx, r, policy.DeleteObjectTaggingAction, opts.VersionID, readErr) } - return authorizeRequestWithExistingTags(ctx, r, policy.DeleteObjectTaggingAction, existingTags) + return authorizeVersionedRequestWithExistingTags(ctx, r, policy.DeleteObjectTaggingAction, opts.VersionID, existingTags) }) if s3Error != ErrNone { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) @@ -4058,9 +4209,9 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r } var s3Error APIErrorCode if isErrObjectNotFound(err) || isErrVersionNotFound(err) { - s3Error = authorizeRequestWithExistingTags(ctx, r, policy.DeleteObjectTaggingAction, "") + s3Error = authorizeVersionedRequestWithExistingTags(ctx, r, policy.DeleteObjectTaggingAction, opts.VersionID, "") } else { - s3Error = authorizeRequestWithUnreadTags(ctx, r, policy.DeleteObjectTaggingAction, err) + s3Error = authorizeVersionedRequestWithUnreadTags(ctx, r, policy.DeleteObjectTaggingAction, opts.VersionID, err) } if s3Error != ErrNone { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) @@ -4071,7 +4222,7 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r } // Allow deleteObjectTagging if policy action is set - if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.DeleteObjectTaggingAction, oi.UserTags); s3Error != ErrNone { + if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.DeleteObjectTaggingAction, opts.VersionID, oi.UserTags); s3Error != ErrNone { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) return } diff --git a/cmd/object-multipart-handlers.go b/cmd/object-multipart-handlers.go index 6ffccf759..75ebb4d6d 100644 --- a/cmd/object-multipart-handlers.go +++ b/cmd/object-multipart-handlers.go @@ -427,7 +427,9 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt } } - if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, srcBucket, srcObject); s3Error != ErrNone { + if s3Error := withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode { + return authorizeVersionedRequestWithPotentialTags(ctx, r, policy.GetObjectAction, vid) + }); s3Error != ErrNone { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) return } @@ -491,7 +493,15 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt } } + var storedSrcTags *string checkCopyPartPrecondFn := func(o ObjectInfo) bool { + storedSrcTags = &o.UserTags + if s3Error := withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode { + return authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, vid, o.UserTags) + }); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return true + } if _, err := DecryptObjectInfo(&o, r); err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return true @@ -513,6 +523,20 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt if isErrPreconditionFailed(err) { return } + var s3Error APIErrorCode + if storedSrcTags != nil { + s3Error = withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode { + return authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, vid, *storedSrcTags) + }) + } else { + s3Error = withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode { + return authorizeVersionedRequestWithUnreadTags(ctx, r, policy.GetObjectAction, vid, err) + }) + } + if s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } if globalBucketVersioningSys.PrefixEnabled(srcBucket, srcObject) && gr != nil { // Versioning enabled quite possibly object is deleted might be delete-marker // if present set the headers, no idea why AWS S3 sets these headers. @@ -521,11 +545,22 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt w.Header()[xhttp.AmzDeleteMarker] = []string{strconv.FormatBool(gr.ObjInfo.DeleteMarker)} } } + reqInfo := logger.GetReqInfo(ctx) + reqInfo.BucketName = srcBucket + reqInfo.ObjectName = srcObject writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return } defer gr.Close() srcInfo := gr.ObjInfo + if storedSrcTags == nil { + if s3Error := withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode { + return authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, vid, srcInfo.UserTags) + }); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + } actualPartSize, err := srcInfo.GetActualSize() if err != nil { @@ -788,10 +823,14 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return } + if size < 0 { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidArgument), r.URL) + return + } } } - if size == -1 { + if size < 0 { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL) return } diff --git a/cmd/policy-import-validation_test.go b/cmd/policy-import-validation_test.go new file mode 100644 index 000000000..3d9bfdb31 --- /dev/null +++ b/cmd/policy-import-validation_test.go @@ -0,0 +1,47 @@ +// Copyright (c) 2026 PGSTY +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cmd + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestIAMImportRejectsUnknownPolicyFields(t *testing.T) { + _, _ = iamAdminFixture(t) + for _, raw := range []string{ + `{"Version":"2012-10-17","Statements":[]}`, + `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::b/*"],"Conditions":{"StringEquals":{"s3:ExistingObjectTag/classification":"public"}}}]}`, + } { + data := iamImportArchive(t, map[string]any{allPoliciesFile: map[string]json.RawMessage{"import-policy": json.RawMessage(raw)}}) + req, err := newTestSignedRequestV4(http.MethodPut, "http://localhost/minio/admin/v3/import-iam-v2", int64(len(data)), bytes.NewReader(data), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil) + mustIAM(t, err) + mustIAM(t, req.ParseForm()) + rec := httptest.NewRecorder() + adminAPIHandlers{}.ImportIAMV2(rec, req.WithContext(newContext(req, rec, "ImportIAM"))) + if rec.Code != http.StatusBadRequest { + t.Fatalf("import malformed policy: got %d %s, want 400", rec.Code, rec.Body.String()) + } + } +} + +func TestSessionPolicyWriteKeepsStrictSemanticValidation(t *testing.T) { + raw := []byte(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:GetBucketQuota"],"Resource":["arn:aws:s3:::mybucket"],"NotResource":["arn:aws:s3:::otherbucket"]}]}`) + if _, err := parseSessionPolicyStrictBytes(raw); err == nil { + t.Fatal("new session policy accepted conflicting Resource and NotResource") + } + if _, err := parsePolicySnapshotBytes(raw); err != nil { + t.Fatalf("historical snapshot compatibility lost: %v", err) + } +} + +func TestBucketPolicyRejectsUnsupportedNotPrincipal(t *testing.T) { + raw := []byte(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","NotPrincipal":{"AWS":"excluded-user"},"Action":"s3:GetObject","Resource":"arn:aws:s3:::b/*"}]}`) + if _, err := parseBucketPolicyStrictBytes(raw, "b"); err == nil { + t.Fatal("unsupported NotPrincipal was silently discarded") + } +} diff --git a/cmd/policy-strict.go b/cmd/policy-strict.go new file mode 100644 index 000000000..ea83541fa --- /dev/null +++ b/cmd/policy-strict.go @@ -0,0 +1,159 @@ +// Copyright (c) 2026 PGSTY +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package cmd + +import ( + "bytes" + "encoding/json" + "io" + + "github.com/pgsty/silo-pkg/v3/policy" +) + +type strictIAMStatementWire struct { + SID policy.ID `json:"Sid,omitempty"` + Effect policy.Effect `json:"Effect"` + Actions json.RawMessage `json:"Action,omitempty"` + NotActions json.RawMessage `json:"NotAction,omitempty"` + Resources json.RawMessage `json:"Resource,omitempty"` + NotResources json.RawMessage `json:"NotResource,omitempty"` + Conditions json.RawMessage `json:"Condition,omitempty"` +} + +type strictIAMPolicyWire struct { + ID policy.ID `json:"ID,omitempty"` + Version string `json:"Version"` + Statements []strictIAMStatementWire `json:"Statement"` +} + +type strictPrincipalWire struct { + AWS json.RawMessage `json:"AWS"` +} + +type strictBucketStatementWire struct { + SID policy.ID `json:"Sid,omitempty"` + Effect policy.Effect `json:"Effect"` + Principal json.RawMessage `json:"Principal,omitempty"` + Actions json.RawMessage `json:"Action,omitempty"` + NotActions json.RawMessage `json:"NotAction,omitempty"` + Resources json.RawMessage `json:"Resource,omitempty"` + NotResources json.RawMessage `json:"NotResource,omitempty"` + Conditions json.RawMessage `json:"Condition,omitempty"` +} + +type strictBucketPolicyWire struct { + ID policy.ID `json:"ID,omitempty"` + Version string `json:"Version"` + Statements []strictBucketStatementWire `json:"Statement"` +} + +func validateStrictIAMPolicyWire(data []byte) ([]byte, error) { + trimmed := bytes.TrimSpace(data) + if len(trimmed) == 0 || trimmed[0] != '{' { + return nil, policy.Errorf("policy must be a JSON object") + } + var wire strictIAMPolicyWire + dec := json.NewDecoder(bytes.NewReader(trimmed)) + dec.DisallowUnknownFields() + if err := dec.Decode(&wire); err != nil { + return nil, policy.Errorf("%w", err) + } + var extra any + if err := dec.Decode(&extra); err != io.EOF { + if err == nil { + return nil, policy.Errorf("unexpected trailing JSON data in policy") + } + return nil, policy.Errorf("%w", err) + } + return trimmed, nil +} + +func parsePolicyStrictBytes(data []byte) (*policy.Policy, error) { + trimmed, err := validateStrictIAMPolicyWire(data) + if err != nil { + return nil, err + } + p, err := policy.ParseConfigStrict(bytes.NewReader(trimmed)) + if err != nil { + return nil, err + } + if p.IsEmpty() { + return nil, policy.Errorf("empty policies are not allowed") + } + return p, nil +} + +func parseSessionPolicyStrictBytes(data []byte) (*policy.Policy, error) { + trimmed, err := validateStrictIAMPolicyWire(data) + if err != nil { + return nil, err + } + return policy.ParseConfigStrict(bytes.NewReader(trimmed)) +} + +// Snapshot imports retain supported historical policy semantics while rejecting +// unknown JSON fields that could silently discard restrictions. +func parsePolicySnapshotBytes(data []byte) (*policy.Policy, error) { + trimmed, err := validateStrictIAMPolicyWire(data) + if err != nil { + return nil, err + } + return policy.ParseConfig(bytes.NewReader(trimmed)) +} + +func validateStrictPrincipalWire(raw json.RawMessage) error { + trimmed := bytes.TrimSpace(raw) + if len(trimmed) == 0 || trimmed[0] != '{' { + return nil + } + var pw strictPrincipalWire + dec := json.NewDecoder(bytes.NewReader(trimmed)) + dec.DisallowUnknownFields() + if err := dec.Decode(&pw); err != nil { + return policy.Errorf("%w", err) + } + var extra any + if err := dec.Decode(&extra); err != io.EOF { + if err == nil { + return policy.Errorf("unexpected trailing JSON data in principal") + } + return policy.Errorf("%w", err) + } + if len(bytes.TrimSpace(pw.AWS)) == 0 { + return policy.Errorf("invalid principal '%s'", string(raw)) + } + return nil +} + +func parseBucketPolicyStrictBytes(data []byte, bucket string) (*policy.BucketPolicy, error) { + trimmed := bytes.TrimSpace(data) + if len(trimmed) == 0 || trimmed[0] != '{' { + return nil, policy.Errorf("bucket policy must be a JSON object") + } + var wire strictBucketPolicyWire + dec := json.NewDecoder(bytes.NewReader(trimmed)) + dec.DisallowUnknownFields() + if err := dec.Decode(&wire); err != nil { + return nil, policy.Errorf("%w", err) + } + var extra any + if err := dec.Decode(&extra); err != io.EOF { + if err == nil { + return nil, policy.Errorf("unexpected trailing JSON data in bucket policy") + } + return nil, policy.Errorf("%w", err) + } + for _, st := range wire.Statements { + if err := validateStrictPrincipalWire(st.Principal); err != nil { + return nil, err + } + } + return policy.ParseBucketPolicyConfig(bytes.NewReader(trimmed), bucket) +} diff --git a/cmd/release-review-boundaries_test.go b/cmd/release-review-boundaries_test.go new file mode 100644 index 000000000..b00f06de4 --- /dev/null +++ b/cmd/release-review-boundaries_test.go @@ -0,0 +1,162 @@ +// Copyright (c) 2026 PGSTY +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cmd + +import ( + "bufio" + "encoding/base64" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/minio/minio/internal/auth" + xhttp "github.com/minio/minio/internal/http" + "github.com/pgsty/silo-pkg/v3/policy" +) + +type trailerFaultReader struct { + err error + reads int +} + +func (r *trailerFaultReader) Read([]byte) (int, error) { + r.reads++ + if r.reads > 2 { + return 0, io.EOF // Bound a broken parser rather than exhausting test memory. + } + return 0, r.err +} + +func TestTrailerReadPreservesConnectionErrors(t *testing.T) { + for _, signed := range []bool{false, true} { + for _, prefix := range []string{"", "checksum:value\r", "checksum:value\r\n", "checksum:value\r\nx-amz-trailer-signature:a\r"} { + if !signed && strings.Contains(prefix, "\r\n") { + continue + } + t.Run(fmt.Sprintf("signed=%t/prefix=%q", signed, prefix), func(t *testing.T) { + failure := errors.New("injected connection reset") + fault := &trailerFaultReader{err: failure} + reader := bufio.NewReader(io.MultiReader(strings.NewReader(prefix), fault)) + var err error + if signed { + err = (&s3ChunkedReader{reader: reader}).readTrailers() + } else { + err = (&s3UnsignedChunkedReader{reader: reader}).readTrailers() + } + if !errors.Is(err, failure) || fault.reads != 1 { + t.Fatalf("got err=%v reads=%d, want original connection error after one read", err, fault.reads) + } + }) + } + } +} + +func TestLatestMetadataListingDoesNotRequireVersionGrants(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, _ auth.Credentials, t *testing.T) { + _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig) + mustIAM(t, err) + _, err = obj.PutObject(t.Context(), bucket, "latest", mustGetPutObjReader(t, strings.NewReader("data"), 4, "", ""), ObjectOptions{ + Versioned: true, UserDefined: map[string]string{xhttp.AmzObjectTagging: "classification=public"}, + }) + mustIAM(t, err) + cred := createTestPolicyUser(t, fmt.Sprintf(`{"Version":"2012-10-17","Statement":[ + {"Effect":"Allow","Action":["s3:ListBucket","s3:ListBucketVersions"],"Resource":["arn:aws:s3:::%s"]}, + {"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectTagging"],"Resource":["arn:aws:s3:::%s/*"]}, + {"Effect":"Deny","Action":["s3:GetObjectVersion","s3:GetObjectVersionTagging"],"Resource":["arn:aws:s3:::%s/*"]}]}`, bucket, bucket, bucket)) + for _, query := range []string{"list-type=2", "versions="} { + req, err := newTestSignedRequestV4(http.MethodGet, "/"+bucket+"?"+query+"&metadata=true&prefix=latest", 0, nil, cred.AccessKey, cred.SecretKey, nil) + mustIAM(t, err) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + wantTags := query == "list-type=2" + if rec.Code != http.StatusOK || strings.Contains(rec.Body.String(), "classification=public") != wantTags { + t.Fatalf("%s %s: status=%d body=%s, want tags=%t", backend, query, rec.Code, rec.Body.String(), wantTags) + } + } + }, + }) +} + +func TestVersionDeniesSurviveInheritedAndSessionPolicies(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, _ auth.Credentials, t *testing.T) { + ctx := t.Context() + _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, enabledBucketVersioningConfig) + mustIAM(t, err) + oi, err := obj.PutObject(ctx, bucket, "item", mustGetPutObjReader(t, strings.NewReader("data"), 4, "", ""), ObjectOptions{ + Versioned: true, UserDefined: map[string]string{xhttp.AmzObjectTagging: "classification=public"}, + }) + mustIAM(t, err) + allow := fmt.Sprintf(`{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectTagging"],"Resource":["arn:aws:s3:::%s/*"]}`, bucket) + deny := fmt.Sprintf(`{"Effect":"Deny","Action":["s3:GetObjectVersion","s3:GetObjectVersionTagging"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:versionid":%q}}}`, bucket, oi.VersionID) + document := func(statements string) string { + return `{"Version":"2012-10-17","Statement":[` + statements + `]}` + } + baseOnly := createTestPolicyUser(t, document(allow)) + denied := createTestPolicyUser(t, document(allow+","+deny)) + inherited, _, err := globalIAMSys.NewServiceAccount(ctx, denied.AccessKey, nil, newServiceAccountOpts{}) + mustIAM(t, err) + session, err := policy.ParseConfig(strings.NewReader(document(allow + "," + deny))) + mustIAM(t, err) + inline, _, err := globalIAMSys.NewServiceAccount(ctx, globalActiveCred.AccessKey, nil, newServiceAccountOpts{sessionPolicy: session}) + mustIAM(t, err) + secret, err := getTokenSigningKey() + mustIAM(t, err) + temporary, err := auth.GetNewCredentialsWithMetadata(map[string]any{ + parentClaim: globalActiveCred.AccessKey, "exp": UTCNow().Add(time.Hour).Unix(), + policy.SessionPolicyName: base64.StdEncoding.EncodeToString([]byte(document(allow + "," + deny))), + }, secret) + mustIAM(t, err) + temporary.ParentUser = globalActiveCred.AccessKey + _, err = globalIAMSys.SetTempUser(ctx, temporary.AccessKey, temporary, "") + mustIAM(t, err) + groupUser := createTestPolicyUser(t, document(allow)) + groupPolicy, err := policy.ParseConfig(strings.NewReader(document(deny))) + mustIAM(t, err) + _, err = globalIAMSys.SetPolicy(ctx, "version-deny", *groupPolicy) + mustIAM(t, err) + _, err = globalIAMSys.AddUsersToGroup(ctx, "version-deny-group", []string{groupUser.AccessKey}) + mustIAM(t, err) + _, err = globalIAMSys.PolicyDBSet(ctx, "version-deny-group", "version-deny", regUser, true) + mustIAM(t, err) + for _, tc := range []struct { + name string + cred auth.Credentials + deny bool + }{ + {"legacy-base-grant", baseOnly, false}, + {"regular", denied, true}, + {"inherited-service-account", inherited, true}, + {"root-service-account-session", inline, true}, + {"root-sts-session", temporary, true}, + {"group", groupUser, true}, + } { + for _, query := range []string{"", "?versionId=" + oi.VersionID, "?tagging=&versionId=" + oi.VersionID} { + headers := map[string]string{xhttp.AmzTagDirective: "ACCESS"} + if tc.cred.IsTemp() { + headers[xhttp.AmzSecurityToken] = tc.cred.SessionToken + } + req, err := newTestSignedRequestV4(http.MethodGet, "/"+bucket+"/item"+query, 0, nil, tc.cred.AccessKey, tc.cred.SecretKey, headers) + mustIAM(t, err) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + want := http.StatusOK + if tc.deny && query != "" { + want = http.StatusForbidden + } + if rec.Code != want { + t.Errorf("%s %s %s: got %d %s, want %d", backend, tc.name, query, rec.Code, rec.Body.String(), want) + } + } + } + }}) +} diff --git a/cmd/release_gate_audit_regressions_test.go b/cmd/release_gate_audit_regressions_test.go new file mode 100644 index 000000000..a69c5bc7d --- /dev/null +++ b/cmd/release_gate_audit_regressions_test.go @@ -0,0 +1,910 @@ +// Copyright (c) 2026 PGSTY +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cmd + +import ( + "archive/zip" + "bytes" + "crypto/hmac" + "crypto/md5" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/bucket/versioning" + "github.com/minio/minio/internal/crypto" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/kms" + "github.com/pgsty/silo-pkg/v3/policy" +) + +func TestStrictPolicyParsingRejectsMalformedAndTypoJSON(t *testing.T) { + invalidDocs := []struct { + name string + raw string + }{ + {"null-literal", "null"}, + {"array-literal", "[]"}, + {"string-literal", `"policy"`}, + {"trailing-object", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::b/*"]}]} {"extra":1}`}, + {"typo-statement-key", `{"Version":"2012-10-17","Statment":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::b/*"]}]}`}, //nolint:misspell + {"unknown-stmt-field", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Actions":["s3:GetObject"],"Resource":["arn:aws:s3:::b/*"]}]}`}, + } + + for _, tc := range invalidDocs { + t.Run(tc.name, func(t *testing.T) { + if _, err := parsePolicyStrictBytes([]byte(tc.raw)); err == nil { + t.Fatalf("parsePolicyStrictBytes(%s) unexpectedly succeeded", tc.name) + } + if _, err := parseSessionPolicyStrictBytes([]byte(tc.raw)); err == nil { + t.Fatalf("parseSessionPolicyStrictBytes(%s) unexpectedly succeeded", tc.name) + } + if _, err := parseBucketPolicyStrictBytes([]byte(tc.raw), "b"); err == nil { + t.Fatalf("parseBucketPolicyStrictBytes(%s) unexpectedly succeeded", tc.name) + } + }) + } + + // Bucket policy must reject unknown Principal keys such as Service instead of treating AWS as empty. + servicePrincipalPolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":["s3.amazonaws.com"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::b/*"]}]}` + if _, err := parseBucketPolicyStrictBytes([]byte(servicePrincipalPolicy), "b"); err == nil { + t.Fatal("parseBucketPolicyStrictBytes accepted Principal.Service without AWS key") + } + + // Explicit session-policy reset documents remain accepted for service accounts, + // while named IAM policies reject empty statement lists. + for _, resetDoc := range []string{`{}`, `{"Statement":[]}`, `{"Version":"2012-10-17","Statement":[]}`} { + sp, err := parseSessionPolicyStrictBytes([]byte(resetDoc)) + if err != nil { + t.Fatalf("parseSessionPolicyStrictBytes(%s) unexpected error: %v", resetDoc, err) + } + if !sp.IsEmpty() { + t.Fatalf("parseSessionPolicyStrictBytes(%s) expected empty policy", resetDoc) + } + if _, err := parsePolicyStrictBytes([]byte(resetDoc)); err == nil { + t.Fatalf("parsePolicyStrictBytes(%s) accepted empty named policy", resetDoc) + } + } + + // Versioning ExcludedPrefixes must be recognized as PrefixesExcluded so Object Lock import rejects it. + var v versioning.Versioning + if err := xml.Unmarshal([]byte(`Enabledtmp/`), &v); err != nil { + t.Fatal(err) + } + if !v.PrefixesExcluded() { + t.Fatal("expected v.PrefixesExcluded() == true when ExcludedPrefixes is populated") + } +} + +func TestLegacyEmptyEmbeddedPolicyServiceAccountStaysDenyAll(t *testing.T) { + f := newGroupRefreshFixture(t) + _, err := f.sys.PolicyDBSet(f.ctx, f.parent.AccessKey, "readwrite", regUser, false) + mustIAM(t, err) + + // Sanity check: parent has full access. + if !f.sys.IsAllowed(policy.Args{AccountName: f.parent.AccessKey, Action: policy.GetObjectAction, BucketName: "b", ObjectName: "k"}) { + t.Fatal("expected parent user to allow GetObject") + } + + const legacyKey = "legacy-deny-all-sa" + legacyClaims := map[string]any{ + iamPolicyClaimNameSA(): embeddedPolicyType, + policy.SessionPolicyName: base64.StdEncoding.EncodeToString([]byte(`{"Version":"2012-10-17","Statement":[]}`)), + } + _, _, err = f.sys.NewServiceAccount(f.ctx, f.parent.AccessKey, nil, newServiceAccountOpts{ + accessKey: legacyKey, + secretKey: "legacy-secret-key-12345", + claims: legacyClaims, + preserveEmptyEmbeddedPolicy: true, + }) + mustIAM(t, err) + mustIAM(t, f.sys.store.LoadIAMCache(f.ctx, false)) + + u, valid, err := f.sys.CheckKey(f.ctx, legacyKey) + mustIAM(t, err) + if !valid { + t.Fatal("expected legacy service account to remain valid") + } + if f.sys.IsAllowed(policy.Args{AccountName: legacyKey, Claims: u.Credentials.Claims, Action: policy.GetObjectAction, BucketName: "b", ObjectName: "k"}) { + t.Fatal("legacy empty embedded-policy service account was promoted to parent permissions") + } + + // Metadata-only edit (nil sessionPolicy) must preserve the deny-all embedded-policy state. + _, err = f.sys.UpdateServiceAccount(f.ctx, legacyKey, updateServiceAccountOpts{ + description: "metadata-only-update", + }) + mustIAM(t, err) + mustIAM(t, f.sys.store.LoadIAMCache(f.ctx, false)) + + u, _, err = f.sys.CheckKey(f.ctx, legacyKey) + mustIAM(t, err) + if u.Credentials.Description != "metadata-only-update" { + t.Fatalf("description = %q, want metadata-only-update", u.Credentials.Description) + } + if f.sys.IsAllowed(policy.Args{AccountName: legacyKey, Claims: u.Credentials.Claims, Action: policy.GetObjectAction, BucketName: "b", ObjectName: "k"}) { + t.Fatal("metadata-only edit promoted legacy deny-all service account to parent permissions") + } + + // Explicit empty reset document clears the inline policy to parent inheritance. + _, err = f.sys.UpdateServiceAccount(f.ctx, legacyKey, updateServiceAccountOpts{ + sessionPolicy: &policy.Policy{}, + }) + mustIAM(t, err) + mustIAM(t, f.sys.store.LoadIAMCache(f.ctx, false)) + + u, _, err = f.sys.CheckKey(f.ctx, legacyKey) + mustIAM(t, err) + if !f.sys.IsAllowed(policy.Args{AccountName: legacyKey, Claims: u.Credentials.Claims, Action: policy.GetObjectAction, BucketName: "b", ObjectName: "k"}) { + t.Fatal("explicit empty-policy reset did not restore parent inheritance") + } +} + +func TestObjectLockSuspendedAndExcludedNullVersionProtection(t *testing.T) { + t.Run("multi-pool-null-version-lock", func(t *testing.T) { + z, bucket := consistencyPools(t) + const key = "null-locked-obj" + retainUntil := UTCNow().Add(24 * time.Hour).Format(time.RFC3339) + + // Pool 0 holds a COMPLIANCE-locked null version; Pool 1 holds an unlocked competing copy. + metaLocked := poolLockMetadata("COMPLIANCE", "OFF", objectLockTestStamp0900, "") + metaLocked[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = retainUntil + metaUnlocked := poolLockMetadata("", "OFF", objectLockTestStamp1000, "") + stamp := UTCNow().Add(-time.Minute) + putConsistencyObject(t, z, bucket, key, 0, "locked-null", ObjectOptions{ + VersionID: nullVersionID, VersionSuspended: true, MTime: stamp, UserDefined: metaLocked, + }) + putConsistencyObject(t, z, bucket, key, 1, "locked-null", ObjectOptions{ + VersionID: nullVersionID, VersionSuspended: true, MTime: stamp.Add(time.Second), UserDefined: metaUnlocked, + }) + + // Unaddressed DeleteObject (prefix-excluded or unversioned delete with RefuseLockedOverwrite) must fail closed, + // even if a newer unlocked UUID version also exists on top of the locked null version. + putConsistencyObject(t, z, bucket, key, 1, "newer-unlocked-v2", ObjectOptions{ + VersionID: mustGetUUID(), Versioned: true, MTime: stamp.Add(2 * time.Second), + }) + _, err := z.DeleteObject(t.Context(), bucket, key, ObjectOptions{RefuseLockedOverwrite: true}) + var lockedErr ObjectLocked + if !errors.As(err, &lockedErr) { + t.Fatalf("DeleteObject error = %v, want ObjectLocked", err) + } + + // Verify the COMPLIANCE-locked null version on pool 0 is untouched. + got, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: nullVersionID, VersionSuspended: true}) + if err != nil { + t.Fatalf("locked null version lost from pool 0: %v", err) + } + if got.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] != "COMPLIANCE" { + t.Fatalf("unexpected retention mode after blocked delete: %+v", got.UserDefined) + } + }) + + t.Run("api-excluded-prefix-unaddressed-delete-and-bulk-delete", func(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + makeBucketOptions: MakeBucketOptions{LockEnabled: true}, + objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) { + const key = "excl/worm-null" + retainUntil := UTCNow().Add(24 * time.Hour).Format(time.RFC3339) + meta := map[string]string{ + strings.ToLower(xhttp.AmzObjectLockMode): "COMPLIANCE", + strings.ToLower(xhttp.AmzObjectLockRetainUntilDate): retainUntil, + } + const payload = "immutable-null-version-data" + // Write a COMPLIANCE-locked null version AND a newer unlocked UUID version on top of it, + // then simulate a legacy Object Lock bucket with ExcludedPrefixes loaded in memory. + _, err := obj.PutObject(t.Context(), bucket, key, mustGetPutObjReader(t, strings.NewReader(payload), int64(len(payload)), "", ""), ObjectOptions{ + Versioned: false, + UserDefined: meta, + }) + if err != nil { + t.Fatalf("%s: PutObject null: %v", backend, err) + } + _, err = obj.PutObject(t.Context(), bucket, key, mustGetPutObjReader(t, strings.NewReader("newer-v2"), int64(len("newer-v2")), "", ""), ObjectOptions{ + Versioned: true, + }) + if err != nil { + t.Fatalf("%s: PutObject v2: %v", backend, err) + } + + bm, err := globalBucketMetadataSys.Get(bucket) + if err != nil { + t.Fatalf("%s: get bucket metadata: %v", backend, err) + } + bm.versioningConfig = &versioning.Versioning{ + XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/", + Status: versioning.Enabled, + ExcludedPrefixes: []versioning.ExcludedPrefix{{Prefix: "excl/"}}, + } + globalBucketMetadataSys.Set(bucket, bm) + + // 1. Unaddressed DELETE /bucket/excl/worm-null must fail closed with ErrObjectLocked (400 InvalidRequest). + delReq, err := newTestSignedRequestV4(http.MethodDelete, "/"+bucket+"/"+key, 0, nil, cred.AccessKey, cred.SecretKey, nil) + if err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + router.ServeHTTP(rec, delReq) + if rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), "WORM protected") { + t.Fatalf("%s: unaddressed DELETE status = %d, want 400 WORM protected; body=%s", backend, rec.Code, rec.Body.String()) + } + + // 2. Unaddressed Multi-Object Delete POST /bucket?delete must report WORM error per key. + delBody := `` + key + `` + sum := md5.Sum([]byte(delBody)) + headers := map[string]string{ + xhttp.ContentMD5: base64.StdEncoding.EncodeToString(sum[:]), + } + bulkReq, err := newTestSignedRequestV4(http.MethodPost, "/"+bucket+"?delete", int64(len(delBody)), strings.NewReader(delBody), cred.AccessKey, cred.SecretKey, headers) + if err != nil { + t.Fatal(err) + } + rec = httptest.NewRecorder() + router.ServeHTTP(rec, bulkReq) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "WORM protected") { + t.Fatalf("%s: DeleteObjects status=%d body=%s, want WORM protected error entry", backend, rec.Code, rec.Body.String()) + } + + // Locked null-version object must remain intact and readable. + if _, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: nullVersionID, VersionSuspended: true}); err != nil { + t.Fatalf("%s: locked null-version object disappeared: %v", backend, err) + } + }, + }) + }) +} + +func TestStreamingTrailerEOFAndNegativeDecodedLength(t *testing.T) { + t.Run("unsigned-trailer-truncated-and-oversized", func(t *testing.T) { + // Truncated mid-trailer (missing final \r\n\r\n) must return ErrUnexpectedEOF, never nil/EOF. + truncatedBody := "5\r\nhello\r\n0\r\nx-amz-checksum-crc32:AAAAAA==" + req := httptest.NewRequest(http.MethodPut, "http://minio.local/b/k", strings.NewReader(truncatedBody)) + req.Header.Set(xhttp.AmzContentSha256, unsignedPayloadTrailer) + req.Header.Set(xhttp.AmzDecodedContentLength, "5") + req.Header.Set(awsTrailerHeader, "x-amz-checksum-crc32") + + rd, code := newUnsignedV4ChunkedReader(req, true) + if code != ErrNone { + t.Fatalf("newUnsignedV4ChunkedReader code = %v", code) + } + _, err := io.ReadAll(rd) + rd.Close() + if !errors.Is(err, io.ErrUnexpectedEOF) { + t.Fatalf("truncated trailer ReadAll error = %v, want io.ErrUnexpectedEOF", err) + } + + // Oversized trailer (> 64 KiB without \r\n\r\n) must reject cleanly without unbounded growth. + oversizedBody := "1\r\na\r\n0\r\nx-amz-checksum-crc32:" + strings.Repeat("A", 64*1024+1024) + req2 := httptest.NewRequest(http.MethodPut, "http://minio.local/b/k", strings.NewReader(oversizedBody)) + req2.Header.Set(xhttp.AmzContentSha256, unsignedPayloadTrailer) + req2.Header.Set(xhttp.AmzDecodedContentLength, "1") + req2.Header.Set(awsTrailerHeader, "x-amz-checksum-crc32") + + rd2, code := newUnsignedV4ChunkedReader(req2, true) + if code != ErrNone { + t.Fatalf("newUnsignedV4ChunkedReader oversized code = %v", code) + } + _, err = io.ReadAll(rd2) + rd2.Close() + if err == nil || errors.Is(err, io.EOF) { + t.Fatalf("oversized trailer ReadAll error = %v, want non-EOF error", err) + } + }) + + t.Run("negative-decoded-content-length-rejected", func(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + endpoints: []string{"PutObject"}, + objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) { + req, err := newStreamingUnsignedTrailerRequest(http.MethodPut, getPutObjectURL("", bucket, "neg-len"), []byte("abc"), UTCNow()) + if err != nil { + t.Fatal(err) + } + req.Header.Set(xhttp.AmzDecodedContentLength, "-5") + if err := signRequestV4(req, cred.AccessKey, cred.SecretKey); err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + if rec.Code != http.StatusBadRequest { + t.Fatalf("%s: negative X-Amz-Decoded-Content-Length status = %d, want 400; body=%s", backend, rec.Code, rec.Body.String()) + } + if _, err := obj.GetObjectInfo(t.Context(), bucket, "neg-len", ObjectOptions{}); err == nil { + t.Fatalf("%s: object stored despite negative X-Amz-Decoded-Content-Length", backend) + } + }, + }) + }) +} + +func signV4WithScopeDate(req *http.Request, accessKey, secretKey, region string, reqTime, scopeTime time.Time, payloadHash string) { + req.Header.Set(xhttp.AmzDate, reqTime.Format(iso8601Format)) + req.Header.Set(xhttp.AmzContentSha256, payloadHash) + if req.Header.Get("Host") == "" { + req.Header.Set("Host", req.Host) + } + signedHeaders := []string{"host", "x-amz-content-sha256", "x-amz-date"} + canonicalHeaders := fmt.Sprintf("host:%s\nx-amz-content-sha256:%s\nx-amz-date:%s\n", + req.Host, payloadHash, reqTime.Format(iso8601Format)) + canonicalRequest := strings.Join([]string{ + req.Method, + req.URL.EscapedPath(), + req.URL.RawQuery, + canonicalHeaders, + strings.Join(signedHeaders, ";"), + payloadHash, + }, "\n") + scope := fmt.Sprintf("%s/%s/s3/aws4_request", scopeTime.Format(yyyymmdd), region) + sum := sha256.Sum256([]byte(canonicalRequest)) + stringToSign := strings.Join([]string{ + signV4Algorithm, + reqTime.Format(iso8601Format), + scope, + hex.EncodeToString(sum[:]), + }, "\n") + signingKey := getSigningKey(secretKey, scopeTime, region, serviceS3) + sig := getSignature(signingKey, stringToSign) + req.Header.Set(xhttp.Authorization, fmt.Sprintf("%s Credential=%s/%s, SignedHeaders=%s, Signature=%s", + signV4Algorithm, accessKey, scope, strings.Join(signedHeaders, ";"), sig)) +} + +func TestSigV4ScopeDateMismatchRejected(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + endpoints: []string{"GetObject", "PutObject"}, + objAPITest: func(_ ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) { + now := UTCNow() + yesterday := now.Add(-24 * time.Hour) + const region = globalMinioDefaultRegion + + // 1. Standard Header SigV4 + req := httptest.NewRequest(http.MethodGet, "http://minio.local/"+bucket+"/obj", nil) + signV4WithScopeDate(req, cred.AccessKey, cred.SecretKey, region, now, yesterday, emptySHA256) + if code := doesSignatureMatch(emptySHA256, req, region, serviceS3); code == ErrNone { + t.Fatalf("%s: doesSignatureMatch accepted mismatched scope date", backend) + } + + // 2. Streaming Seed SigV4 + sreq := httptest.NewRequest(http.MethodPut, "http://minio.local/"+bucket+"/obj", strings.NewReader("0\r\n\r\n")) + sreq.Header.Set(xhttp.AmzDecodedContentLength, "0") + signV4WithScopeDate(sreq, cred.AccessKey, cred.SecretKey, region, now, yesterday, streamingContentSHA256) + if _, _, _, _, code := calculateSeedSignature(sreq, false); code == ErrNone { + t.Fatalf("%s: calculateSeedSignature accepted mismatched scope date", backend) + } + + // 3. Presigned SigV4 + q := url.Values{} + scope := fmt.Sprintf("%s/%s/s3/aws4_request", yesterday.Format(yyyymmdd), region) + q.Set(xhttp.AmzAlgorithm, signV4Algorithm) + q.Set(xhttp.AmzCredential, cred.AccessKey+"/"+scope) + q.Set(xhttp.AmzDate, now.Format(iso8601Format)) + q.Set(xhttp.AmzExpires, "300") + q.Set(xhttp.AmzSignedHeaders, "host") + preq := httptest.NewRequest(http.MethodGet, "http://minio.local/"+bucket+"/obj?"+q.Encode(), nil) + canonicalPresigned := strings.Join([]string{ + http.MethodGet, + preq.URL.EscapedPath(), + preq.URL.Query().Encode(), + "host:" + preq.Host + "\n", + "host", + unsignedPayload, + }, "\n") + psum := sha256.Sum256([]byte(canonicalPresigned)) + pSTS := strings.Join([]string{ + signV4Algorithm, + now.Format(iso8601Format), + scope, + hex.EncodeToString(psum[:]), + }, "\n") + pSig := getSignature(getSigningKey(cred.SecretKey, yesterday, region, serviceS3), pSTS) + q.Set(xhttp.AmzSignature, pSig) + preq.URL.RawQuery = q.Encode() + if code := doesPresignedSignatureMatch(unsignedPayload, preq, region, serviceS3); code == ErrNone { + t.Fatalf("%s: doesPresignedSignatureMatch accepted mismatched scope date", backend) + } + + // 4. POST Policy SigV4 + policyDoc := fmt.Sprintf(`{"expiration":%q,"conditions":[["eq","$bucket",%q],["eq","$x-amz-date",%q],["eq","$x-amz-credential",%q],["eq","$x-amz-algorithm","AWS4-HMAC-SHA256"]]}`, + now.Add(10*time.Minute).Format(iso8601Format), bucket, now.Format(iso8601Format), cred.AccessKey+"/"+scope) + policyB64 := base64.StdEncoding.EncodeToString([]byte(policyDoc)) + mac := hmac.New(sha256.New, getSigningKey(cred.SecretKey, yesterday, region, serviceS3)) + mac.Write([]byte(policyB64)) + postSig := hex.EncodeToString(mac.Sum(nil)) + formValues := http.Header{} + formValues.Set(xhttp.AmzAlgorithm, signV4Algorithm) + formValues.Set(xhttp.AmzCredential, cred.AccessKey+"/"+scope) + formValues.Set(xhttp.AmzDate, now.Format(iso8601Format)) + formValues.Set(xhttp.AmzSignature, postSig) + formValues.Set("Policy", policyB64) + if _, code := doesPolicySignatureV4Match(formValues); code == ErrNone { + t.Fatalf("%s: doesPolicySignatureV4Match accepted mismatched scope date", backend) + } + }, + }) +} + +func TestBatchKeyRotateLegacyXLV1AndSSES3ToKMS(t *testing.T) { + f := newBatchAuthzFixture(t) + previous := GlobalKMS + t.Cleanup(func() { GlobalKMS = previous }) + configured, err := kms.NewBuiltin("rotate-key", bytes.Repeat([]byte{0x37}, 32)) + mustIAM(t, err) + GlobalKMS = configured + + t.Run("sse-s3-multipart-to-sse-kms-empty-context", func(t *testing.T) { + const object = "sses3-mp/item" + const payload = "multipart sse-s3 migrated to sse-kms without losing multipart flag" + metadata := map[string]string{ + crypto.MetaMultipart: "1", + } + key, err := newEncryptMetadata(f.ctx, crypto.S3, "", nil, f.bucket, object, metadata, nil) + mustIAM(t, err) + ciphertext, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader(payload), key)) + mustIAM(t, err) + original, err := f.obj.PutObject(f.ctx, f.bucket, object, mustGetPutObjReader(t, bytes.NewReader(ciphertext), int64(len(ciphertext)), "", ""), ObjectOptions{ + Versioned: true, + UserDefined: metadata, + }) + mustIAM(t, err) + + rec, job := f.submit(t, globalActiveCred, fmt.Sprintf("keyrotate:\n apiVersion: v1\n bucket: %s\n prefix: sses3-mp/\n encryption:\n type: sse-kms\n key: rotate-key\n", f.bucket)) + if job == nil { + t.Fatalf("submit: %d %s", rec.Code, rec.Body.String()) + } + mustIAM(t, job.KeyRotate.Start(f.ctx, f.obj, *job)) + progress := f.jobInfo(t, job) + if !progress.Complete || progress.Failed || progress.Objects != 1 { + t.Fatalf("SSE-S3 -> SSE-KMS rotation failed: %+v", progress) + } + + after, err := f.obj.GetObjectInfo(f.ctx, f.bucket, object, ObjectOptions{VersionID: original.VersionID}) + mustIAM(t, err) + if !crypto.S3KMS.IsEncrypted(after.UserDefined) || crypto.S3.IsEncrypted(after.UserDefined) { + t.Fatalf("unexpected encryption headers after SSE-S3 -> SSE-KMS: %+v", after.UserDefined) + } + if after.UserDefined[crypto.MetaMultipart] != "1" { + t.Fatalf("MetaMultipart stripped during SSE-S3 -> SSE-KMS rotation: %+v", after.UserDefined) + } + actual, err := crypto.S3KMS.UnsealObjectKey(configured, after.UserDefined, f.bucket, object) + mustIAM(t, err) + if actual != key { + t.Fatal("data encryption key changed during metadata rotation") + } + }) + + t.Run("legacy-xlv1-stream-rewrite-rotation", func(t *testing.T) { + const legacyBucket = "legacy-xlv1-bucket" + mustIAM(t, f.obj.MakeBucket(f.ctx, legacyBucket, MakeBucketOptions{})) + + const object = "legacy/obj" + payload := strings.Repeat("legacy XLV1 object rotated cleanly to XLV2 without panic! ", 4096) + metadata := map[string]string{ + xhttp.AmzObjectTagging: "tier=legacy", + } + key, err := newEncryptMetadata(f.ctx, crypto.S3KMS, "rotate-key", nil, legacyBucket, object, metadata, kms.Context{"stage": "v1"}) + mustIAM(t, err) + ciphertext, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader(payload), key)) + mustIAM(t, err) + plainMD5 := md5.Sum([]byte(payload)) + sealedETag := hex.EncodeToString(key.SealETag(plainMD5[:])) + metadata["etag"] = sealedETag + + _, err = f.obj.PutObject(f.ctx, legacyBucket, object, mustGetPutObjReader(t, bytes.NewReader(ciphertext), int64(len(ciphertext)), "", ""), ObjectOptions{ + UserDefined: metadata, + }) + mustIAM(t, err) + + // Convert stored XLV2 layout on the single test disk into a real XLV1 xl.json + part.1 layout. + disk := f.obj.(*erasureServerPools).serverPools[0].sets[0].getDisks()[0] + xlDisk := disk.(*xlStorageDiskIDCheck).storage + xlDisk.Lock() + xlDisk.formatLegacy = true + xlDisk.Unlock() + t.Cleanup(func() { + xlDisk.Lock() + xlDisk.formatLegacy = false + xlDisk.Unlock() + }) + + fi, err := disk.ReadVersion(f.ctx, "", legacyBucket, object, "", ReadOptions{}) + mustIAM(t, err) + + partPath := pathJoin(object, fi.DataDir, "part.1") + rawPartBytes, err := disk.ReadAll(f.ctx, legacyBucket, partPath) + mustIAM(t, err) + + mustIAM(t, disk.Delete(f.ctx, legacyBucket, pathJoin(object, xlStorageFormatFile), DeleteOptions{})) + mustIAM(t, disk.Delete(f.ctx, legacyBucket, partPath, DeleteOptions{})) + mustIAM(t, disk.AppendFile(f.ctx, legacyBucket, pathJoin(object, "part.1"), rawPartBytes)) + + fi.Metadata["etag"] = sealedETag + v1Obj := xlMetaV1Object{ + Version: xlMetaVersion101, + Format: xlMetaFormat, + Stat: StatInfo{ + Size: fi.Size, + ModTime: fi.ModTime, + }, + Erasure: fi.Erasure, + Meta: fi.Metadata, + Parts: fi.Parts, + } + v1Obj.Minio.Release = "RELEASE.2019-12-30T05-45-39Z" + v1Bytes, err := json.Marshal(v1Obj) + mustIAM(t, err) + mustIAM(t, disk.AppendFile(f.ctx, legacyBucket, pathJoin(object, xlStorageFormatFileV1), v1Bytes)) + + legacyOI, err := f.obj.GetObjectInfo(f.ctx, legacyBucket, object, ObjectOptions{}) + mustIAM(t, err) + if !legacyOI.Legacy { + t.Fatal("expected test fixture object to report Legacy (XLV1) == true") + } + + newCtxB64 := base64.StdEncoding.EncodeToString([]byte(`{"stage":"v2"}`)) + rec, job := f.submit(t, globalActiveCred, fmt.Sprintf("keyrotate:\n apiVersion: v1\n bucket: %s\n prefix: legacy/\n encryption:\n type: sse-kms\n key: rotate-key\n context: %s\n", legacyBucket, newCtxB64)) + if job == nil { + t.Fatalf("submit legacy keyrotate: %d %s", rec.Code, rec.Body.String()) + } + mustIAM(t, job.KeyRotate.Start(f.ctx, f.obj, *job)) + progress := f.jobInfo(t, job) + if !progress.Complete || progress.Failed || progress.Objects != 1 { + t.Fatalf("legacy XLV1 rotation failed: %+v", progress) + } + + after, err := f.obj.GetObjectInfo(f.ctx, legacyBucket, object, ObjectOptions{}) + mustIAM(t, err) + if after.Legacy { + t.Fatal("expected rotated legacy object to be upgraded to XLV2 (Legacy == false)") + } + if after.UserTags != "tier=legacy" { + t.Fatalf("UserTags = %q, want tier=legacy", after.UserTags) + } + if got := readCopyChecksumObject(t, f.obj, legacyBucket, object, ObjectOptions{}); string(got) != payload { + t.Fatalf("decrypted content = %q, want %q", string(got), payload) + } + newObjKey, err := crypto.S3KMS.UnsealObjectKey(configured, after.UserDefined, legacyBucket, object) + mustIAM(t, err) + unsealedETag, err := DecryptETag(newObjKey, ObjectInfo{ETag: after.ETag}) + mustIAM(t, err) + if wantETag := hex.EncodeToString(plainMD5[:]); unsealedETag != wantETag { + t.Fatalf("unsealed ETag = %q, want %q", unsealedETag, wantETag) + } + }) +} + +func createTestPolicyUser(t *testing.T, policyJSON string) auth.Credentials { + t.Helper() + ctx := t.Context() + ak, sk, err := auth.GenerateCredentials() + if err != nil { + t.Fatal(err) + } + creds := auth.Credentials{AccessKey: ak, SecretKey: sk} + if _, err = globalIAMSys.CreateUser(ctx, creds.AccessKey, madmin.AddOrUpdateUserReq{ + SecretKey: creds.SecretKey, + Status: madmin.AccountEnabled, + }); err != nil { + t.Fatal(err) + } + parsed, err := policy.ParseConfig(strings.NewReader(policyJSON)) + if err != nil { + t.Fatalf("parse policy: %v\n%s", err, policyJSON) + } + pName := "audit-policy-" + mustGetUUID() + if _, err = globalIAMSys.SetPolicy(ctx, pName, *parsed); err != nil { + t.Fatal(err) + } + if _, err = globalIAMSys.PolicyDBSet(ctx, creds.AccessKey, pName, regUser, false); err != nil { + t.Fatal(err) + } + return creds +} + +func TestObjectTagAndVersionAuthSymmetry(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, adminCred auth.Credentials, t *testing.T) { + if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig); err != nil { + t.Fatalf("%s: enable versioning: %v", backend, err) + } + + // Create v1 (env=prod) and v2 (env=dev) of src-obj. + v1, err := obj.PutObject(t.Context(), bucket, "src-obj", mustGetPutObjReader(t, strings.NewReader("prod-secret"), int64(len("prod-secret")), "", ""), ObjectOptions{ + Versioned: true, + UserDefined: map[string]string{xhttp.AmzObjectTagging: "env=prod"}, + }) + if err != nil { + t.Fatal(err) + } + v2, err := obj.PutObject(t.Context(), bucket, "src-obj", mustGetPutObjReader(t, strings.NewReader("dev-public"), int64(len("dev-public")), "", ""), ObjectOptions{ + Versioned: true, + UserDefined: map[string]string{xhttp.AmzObjectTagging: "env=dev"}, + }) + if err != nil { + t.Fatal(err) + } + + // Also create a ZIP object with env=prod tag to verify ZIP sub-handlers enforce ExistingObjectTag. + var zipBuf bytes.Buffer + zw := zip.NewWriter(&zipBuf) + zf, err := zw.Create("inside.txt") + if err != nil { + t.Fatal(err) + } + _, _ = zf.Write([]byte("zip-inner")) + if err := zw.Close(); err != nil { + t.Fatal(err) + } + _, err = obj.PutObject(t.Context(), bucket, "archive.zip", mustGetPutObjReader(t, bytes.NewReader(zipBuf.Bytes()), int64(zipBuf.Len()), "", ""), ObjectOptions{ + Versioned: true, + UserDefined: map[string]string{xhttp.AmzObjectTagging: "env=prod"}, + }) + if err != nil { + t.Fatal(err) + } + + // User A: + // - Can read/tag ONLY objects/versions tagged env=dev. + // - Can write dst/* ONLY when RequestObjectTag/env == dev (plus unconditional PutObject on prod-readable copy test). + userAPolicy := fmt.Sprintf(`{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:GetObjectVersion", + "s3:GetObjectTagging", + "s3:GetObjectVersionTagging", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging", + "s3:DeleteObjectTagging", + "s3:DeleteObjectVersionTagging" + ], + "Resource": ["arn:aws:s3:::%s/*"], + "Condition": {"StringEquals": {"s3:ExistingObjectTag/env": ["dev"]}} + }, + { + "Effect": "Allow", + "Action": ["s3:ListBucket", "s3:ListBucketVersions", "s3:ListBucketMultipartUploads"], + "Resource": ["arn:aws:s3:::%s"] + }, + { + "Effect": "Allow", + "Action": ["s3:PutObject"], + "Resource": ["arn:aws:s3:::%s/dst-*"], + "Condition": {"StringEquals": {"s3:RequestObjectTag/env": ["dev"]}} + }, + { + "Effect": "Allow", + "Action": ["s3:PutObject", "s3:AbortMultipartUpload", "s3:ListMultipartUploadParts"], + "Resource": ["arn:aws:s3:::%s/mp-*"] + } + ] + }`, bucket, bucket, bucket, bucket) + userA := createTestPolicyUser(t, userAPolicy) + + // 1. CopyObject from v1 (env=prod) must fail 403 due to source ExistingObjectTag. + cpDenied, err := newTestSignedRequestV4(http.MethodPut, "/"+bucket+"/dst-denied", 0, nil, userA.AccessKey, userA.SecretKey, map[string]string{ + xhttp.AmzCopySource: "/" + bucket + "/src-obj?versionId=" + v1.VersionID, + xhttp.AmzTagDirective: "REPLACE", + xhttp.AmzObjectTagging: "env=dev", + }) + if err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + router.ServeHTTP(rec, cpDenied) + if rec.Code != http.StatusForbidden { + t.Fatalf("%s: CopyObject from env=prod version status=%d, want 403", backend, rec.Code) + } + + // 2. CopyObject from v2 (env=dev) with COPY directive succeeds (RequestObjectTag/env == dev from copied tags). + cpOK, err := newTestSignedRequestV4(http.MethodPut, "/"+bucket+"/dst-ok", 0, nil, userA.AccessKey, userA.SecretKey, map[string]string{ + xhttp.AmzCopySource: "/" + bucket + "/src-obj?versionId=" + v2.VersionID, + xhttp.AmzTagDirective: "COPY", + }) + if err != nil { + t.Fatal(err) + } + rec = httptest.NewRecorder() + router.ServeHTTP(rec, cpOK) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CopyObject from env=dev with COPY directive status=%d body=%s, want 200", backend, rec.Code, rec.Body.String()) + } + + // 3. RequestObjectTag spoofing defense on COPY directive: + // User B can read all versions of src-obj, but can PutObject to dst-* ONLY when RequestObjectTag/env == dev. + // Copying v1 (env=prod) with Tagging-Directive: COPY and a spoofed X-Amz-Tagging: env=dev header MUST fail 403! + userBPolicy := fmt.Sprintf(`{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:GetObjectVersion", "s3:GetObjectTagging", "s3:GetObjectVersionTagging"], + "Resource": ["arn:aws:s3:::%s/src-obj"] + }, + { + "Effect": "Allow", + "Action": ["s3:PutObject", "s3:PutObjectTagging"], + "Resource": ["arn:aws:s3:::%s/dst-*"], + "Condition": {"StringEquals": {"s3:RequestObjectTag/env": ["dev"]}} + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject"], + "Resource": ["arn:aws:s3:::%s/src-obj"] + } + ] + }`, bucket, bucket, bucket) + userB := createTestPolicyUser(t, userBPolicy) + + cpSpoof, err := newTestSignedRequestV4(http.MethodPut, "/"+bucket+"/dst-spoof", 0, nil, userB.AccessKey, userB.SecretKey, map[string]string{ + xhttp.AmzCopySource: "/" + bucket + "/src-obj?versionId=" + v1.VersionID, + xhttp.AmzTagDirective: "COPY", + xhttp.AmzObjectTagging: "env=dev", + }) + if err != nil { + t.Fatal(err) + } + rec = httptest.NewRecorder() + router.ServeHTTP(rec, cpSpoof) + if rec.Code != http.StatusForbidden { + t.Fatalf("%s: CopyObject accepted spoofed X-Amz-Tagging under COPY directive: status=%d body=%s", backend, rec.Code, rec.Body.String()) + } + + // 4. UploadPartCopy enforces ExistingObjectTag on versioned source! + initMP, err := newTestSignedRequestV4(http.MethodPost, "/"+bucket+"/mp-target?uploads", 0, nil, userA.AccessKey, userA.SecretKey, nil) + if err != nil { + t.Fatal(err) + } + rec = httptest.NewRecorder() + router.ServeHTTP(rec, initMP) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CreateMultipartUpload status=%d body=%s", backend, rec.Code, rec.Body.String()) + } + var initResp InitiateMultipartUploadResponse + if err := xml.Unmarshal(rec.Body.Bytes(), &initResp); err != nil { + t.Fatal(err) + } + + upcDenied, err := newTestSignedRequestV4(http.MethodPut, fmt.Sprintf("/%s/mp-target?partNumber=1&uploadId=%s", bucket, url.QueryEscape(initResp.UploadID)), 0, nil, userA.AccessKey, userA.SecretKey, map[string]string{ + xhttp.AmzCopySource: "/" + bucket + "/src-obj?versionId=" + v1.VersionID, + }) + if err != nil { + t.Fatal(err) + } + rec = httptest.NewRecorder() + router.ServeHTTP(rec, upcDenied) + if rec.Code != http.StatusForbidden { + t.Fatalf("%s: UploadPartCopy from env=prod status=%d, want 403", backend, rec.Code) + } + + upcOK, err := newTestSignedRequestV4(http.MethodPut, fmt.Sprintf("/%s/mp-target?partNumber=1&uploadId=%s", bucket, url.QueryEscape(initResp.UploadID)), 0, nil, userA.AccessKey, userA.SecretKey, map[string]string{ + xhttp.AmzCopySource: "/" + bucket + "/src-obj?versionId=" + v2.VersionID, + }) + if err != nil { + t.Fatal(err) + } + rec = httptest.NewRecorder() + router.ServeHTTP(rec, upcOK) + if rec.Code != http.StatusOK { + t.Fatalf("%s: UploadPartCopy from env=dev status=%d body=%s, want 200", backend, rec.Code, rec.Body.String()) + } + + // 5. GET / HEAD with X-Amz-Tagging-Directive: ACCESS must NOT leak X-Amz-Tagging when GetObjectTagging is denied! + noTagReadPolicy := fmt.Sprintf(`{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:GetObjectVersion"], + "Resource": ["arn:aws:s3:::%s/*"] + }] + }`, bucket) + userNoTag := createTestPolicyUser(t, noTagReadPolicy) + for _, method := range []string{http.MethodGet, http.MethodHead} { + req, err := newTestSignedRequestV4(method, "/"+bucket+"/src-obj?versionId="+v1.VersionID, 0, nil, userNoTag.AccessKey, userNoTag.SecretKey, map[string]string{ + xhttp.AmzTagDirective: "ACCESS", + }) + if err != nil { + t.Fatal(err) + } + rec = httptest.NewRecorder() + router.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("%s: %s status=%d, want 200", backend, method, rec.Code) + } + if gotTag := rec.Header().Get(xhttp.AmzObjectTagging); gotTag != "" { + t.Fatalf("%s: %s leaked X-Amz-Tagging=%q without GetObjectVersionTagging permission", backend, method, gotTag) + } + if gotCnt := rec.Header().Get(xhttp.AmzTagCount); gotCnt != "" { + t.Fatalf("%s: %s leaked X-Amz-Tagging-Count=%q without GetObjectVersionTagging permission", backend, method, gotCnt) + } + } + + // 6. ListObjectVersions with metadata=true filters per version's ExistingObjectTag! + listVerReq, err := newTestSignedRequestV4(http.MethodGet, "/"+bucket+"?versions=&metadata=true&prefix=src-obj", 0, nil, userA.AccessKey, userA.SecretKey, nil) + if err != nil { + t.Fatal(err) + } + rec = httptest.NewRecorder() + router.ServeHTTP(rec, listVerReq) + if rec.Code != http.StatusOK { + t.Fatalf("%s: ListObjectVersions status=%d body=%s", backend, rec.Code, rec.Body.String()) + } + bodyStr := rec.Body.String() + if strings.Contains(bodyStr, "env=prod") { + t.Fatalf("%s: ListObjectVersions metadata=true leaked unauthorized version tags env=prod: %s", backend, bodyStr) + } + if !strings.Contains(bodyStr, "env=dev") { + t.Fatalf("%s: ListObjectVersions metadata=true omitted authorized version tags env=dev: %s", backend, bodyStr) + } + + // 7. GetObjectTagging & DeleteObjectTagging with ?versionId= enforce ExistingObjectTag on that version! + for _, method := range []string{http.MethodGet, http.MethodDelete} { + tagDenied, err := newTestSignedRequestV4(method, "/"+bucket+"/src-obj?tagging=&versionId="+v1.VersionID, 0, nil, userA.AccessKey, userA.SecretKey, nil) + if err != nil { + t.Fatal(err) + } + rec = httptest.NewRecorder() + router.ServeHTTP(rec, tagDenied) + if rec.Code != http.StatusForbidden { + t.Fatalf("%s: %s tagging on v1 (env=prod) status=%d, want 403", backend, method, rec.Code) + } + } + getTagOK, err := newTestSignedRequestV4(http.MethodGet, "/"+bucket+"/src-obj?tagging=&versionId="+v2.VersionID, 0, nil, userA.AccessKey, userA.SecretKey, nil) + if err != nil { + t.Fatal(err) + } + rec = httptest.NewRecorder() + router.ServeHTTP(rec, getTagOK) + if rec.Code != http.StatusOK { + t.Fatalf("%s: GET tagging on v2 (env=dev) status=%d, want 200", backend, rec.Code) + } + + // 8. Archive x-minio-extract sub-handlers (HEAD/GET/ListObjectsV2) reject access when archive ExistingObjectTag mismatches! + for _, tc := range []struct { + name, method, target string + }{ + {"zip-get", http.MethodGet, "/" + bucket + "/archive.zip/inside.txt"}, + {"zip-head", http.MethodHead, "/" + bucket + "/archive.zip/inside.txt"}, + {"zip-list", http.MethodGet, "/" + bucket + "?list-type=2&prefix=archive.zip/"}, + } { + zreq, err := newTestSignedRequestV4(tc.method, tc.target, 0, nil, userA.AccessKey, userA.SecretKey, map[string]string{ + "x-minio-extract": "true", + }) + if err != nil { + t.Fatal(err) + } + rec = httptest.NewRecorder() + router.ServeHTTP(rec, zreq) + if rec.Code != http.StatusForbidden { + t.Fatalf("%s: %s on env=prod archive status=%d, want 403", backend, tc.name, rec.Code) + } + } + + // Clean up multipart upload with admin credentials so DetectTestLeak sees no background state. + abortReq, err := newTestSignedRequestV4(http.MethodDelete, fmt.Sprintf("/%s/mp-target?uploadId=%s", bucket, url.QueryEscape(initResp.UploadID)), 0, nil, adminCred.AccessKey, adminCred.SecretKey, nil) + if err == nil { + router.ServeHTTP(httptest.NewRecorder(), abortReq) + } + }, + }) +} diff --git a/cmd/s3-zip-handlers.go b/cmd/s3-zip-handlers.go index 068b1b813..15a8caa56 100644 --- a/cmd/s3-zip-handlers.go +++ b/cmd/s3-zip-handlers.go @@ -102,7 +102,7 @@ func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context, // Validate pre-conditions if any. opts.CheckPrecondFn = func(oi ObjectInfo) bool { - if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.GetObjectAction, oi.UserTags); s3Error != ErrNone { + if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, opts.VersionID, oi.UserTags); s3Error != ErrNone { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) return true } @@ -116,7 +116,7 @@ func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context, zipObjInfo, err := getObjectInfo(ctx, bucket, zipPath, opts) if err != nil { - err = archiveReadError(ctx, r, bucket, zipPath, err) + err = archiveReadError(ctx, r, bucket, zipPath, opts.VersionID, err) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return } @@ -181,6 +181,7 @@ func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context, defer rc.Close() + opts.Tagging = false if err = setObjectHeaders(ctx, w, fileObjInfo, nil, opts); err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return @@ -224,7 +225,7 @@ func listObjectsV2InArchive(ctx context.Context, objectAPI ObjectLayer, bucket, opts := ObjectOptions{} zipObjInfo, err := objectAPI.GetObjectInfo(ctx, bucket, zipPath, opts) if err != nil { - if s3Error := authorizeRequestWithUnreadTags(ctx, r, policy.GetObjectAction, err); s3Error != ErrNone { + if s3Error := authorizeVersionedRequestWithUnreadTags(ctx, r, policy.GetObjectAction, "", err); s3Error != ErrNone { return ListObjectsV2Info{}, PrefixAccessDenied{Bucket: bucket, Object: zipPath} } if isErrObjectNotFound(err) { @@ -232,12 +233,12 @@ func listObjectsV2InArchive(ctx context.Context, objectAPI ObjectLayer, bucket, } return ListObjectsV2Info{}, err } - if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.GetObjectAction, zipObjInfo.UserTags); s3Error != ErrNone { + if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, "", zipObjInfo.UserTags); s3Error != ErrNone { return ListObjectsV2Info{}, PrefixAccessDenied{Bucket: bucket, Object: zipPath} } var denied bool opts.CheckPrecondFn = func(oi ObjectInfo) bool { - denied = authorizeRequestWithExistingTags(ctx, r, policy.GetObjectAction, oi.UserTags) != ErrNone + denied = authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, "", oi.UserTags) != ErrNone return denied } zipInfo := zipObjInfo.ArchiveInfo(r.Header) @@ -387,7 +388,7 @@ func (api objectAPIHandlers) headObjectInArchiveFileHandler(ctx context.Context, // Validate pre-conditions if any. opts.CheckPrecondFn = func(oi ObjectInfo) bool { - if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.GetObjectAction, oi.UserTags); s3Error != ErrNone { + if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, opts.VersionID, oi.UserTags); s3Error != ErrNone { writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(s3Error)) return true } @@ -407,7 +408,7 @@ func (api objectAPIHandlers) headObjectInArchiveFileHandler(ctx context.Context, zipObjInfo, err := getObjectInfo(ctx, bucket, zipPath, opts) if err != nil { - err = archiveReadError(ctx, r, bucket, zipPath, err) + err = archiveReadError(ctx, r, bucket, zipPath, opts.VersionID, err) writeErrorResponseHeadersOnly(w, toAPIError(ctx, err)) return } @@ -444,6 +445,7 @@ func (api objectAPIHandlers) headObjectInArchiveFileHandler(ctx context.Context, ModTime: zipObjInfo.ModTime, } + opts.Tagging = false // Set standard object headers. if err = setObjectHeaders(ctx, w, objInfo, nil, opts); err != nil { writeErrorResponseHeadersOnly(w, toAPIError(ctx, err)) @@ -462,12 +464,12 @@ func (api objectAPIHandlers) headObjectInArchiveFileHandler(ctx context.Context, // archiveReadError preserves the anonymous ListBucket missing-object response. // All other failures are disclosed only to callers authorized to read the ZIP. -func archiveReadError(ctx context.Context, r *http.Request, bucket, object string, err error) error { +func archiveReadError(ctx context.Context, r *http.Request, bucket, object, versionID string, err error) error { if getRequestAuthType(r) == authTypeAnonymous && isErrObjectNotFound(err) && authorizeBucketRequest(ctx, r, policy.ListBucketAction) == ErrNone { return err } - if authorizeRequestWithUnreadTags(ctx, r, policy.GetObjectAction, err) != ErrNone { + if authorizeVersionedRequestWithUnreadTags(ctx, r, policy.GetObjectAction, versionID, err) != ErrNone { return PrefixAccessDenied{Bucket: bucket, Object: object} } return err diff --git a/cmd/signature-v4.go b/cmd/signature-v4.go index d0125fb1c..c27ec7da1 100644 --- a/cmd/signature-v4.go +++ b/cmd/signature-v4.go @@ -189,6 +189,16 @@ func doesPolicySignatureV4Match(formValues http.Header) (auth.Credentials, APIEr return cred, s3Err } + if dateStr := formValues.Get(xhttp.AmzDate); dateStr != "" { + t, err := time.Parse(iso8601Format, dateStr) + if err != nil { + return cred, ErrMalformedDate + } + if credHeader.scope.date.Format(yyyymmdd) != t.UTC().Format(yyyymmdd) { + return cred, ErrSignatureDoesNotMatch + } + } + // Get signing key. signingKey := getSigningKey(cred.SecretKey, credHeader.scope.date, credHeader.scope.region, serviceS3) @@ -248,6 +258,9 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s // Save the date and expires. t := pSignValues.Date + if pSignValues.Credential.scope.date.Format(yyyymmdd) != t.UTC().Format(yyyymmdd) { + return ErrSignatureDoesNotMatch + } expireSeconds := int(pSignValues.Expires / time.Second) // Construct new query. @@ -388,6 +401,9 @@ func doesSignatureMatch(hashedPayload string, r *http.Request, region string, st if e != nil { return ErrMalformedDate } + if signV4Values.Credential.scope.date.Format(yyyymmdd) != t.UTC().Format(yyyymmdd) { + return ErrSignatureDoesNotMatch + } // Query string. queryStr := req.Form.Encode() diff --git a/cmd/site-replication-metadata_test.go b/cmd/site-replication-metadata_test.go index de69b2b50..91a718c49 100644 --- a/cmd/site-replication-metadata_test.go +++ b/cmd/site-replication-metadata_test.go @@ -622,6 +622,23 @@ func TestLocalBucketMetadataCommittedEvents(t *testing.T) { t.Fatalf("zero quota event: %+v", last) } + // Replicated buckets must reject prefix exclusions without changing + // the stored versioning document or announcing a successful update. + beforeRejected := len(events()) + rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata", corsZip(t, map[string][]byte{ + bucket + "/" + bucketVersioningConfig: []byte(`Enabledtrue`), + })) + if report := corsImportReport(t, rec).Buckets[bucket]; report.Versioning.Err == "" { + t.Fatal("replicated bucket accepted excluded prefixes") + } + afterRejected, err := loadBucketMetadata(ctx, obj, bucket) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(afterRejected.VersioningConfigXML, meta.VersioningConfigXML) || !afterRejected.VersioningConfigUpdatedAt.Equal(meta.VersioningConfigUpdatedAt) || len(events()) != beforeRejected { + t.Fatal("rejected versioning import changed metadata or emitted an event") + } + injectedAt := UTCNow().Add(2 * time.Hour) interleaving := &importInterleavingObjectLayer{ObjectLayer: obj, write: func() { data := []byte(`{"quota":2048,"quotatype":"hard"}`) @@ -638,10 +655,10 @@ func TestLocalBucketMetadataCommittedEvents(t *testing.T) { bucket + "/quota.json": []byte(`{}`), bucket + "/" + bucketTaggingConfig: []byte(`importedyes`), bucket + "/" + objectLockConfig: enabledBucketObjectLockConfig, - bucket + "/" + bucketVersioningConfig: []byte(`Enabledtrue`), + bucket + "/" + bucketVersioningConfig: enabledBucketVersioningConfig, })) report := corsImportReport(t, rec).Buckets[bucket] - if report.Err != "" || report.Policy.Err != "" || report.Quota.Err != "" || report.Tagging.Err != "" { + if report.Err != "" || report.Policy.Err != "" || report.Quota.Err != "" || report.Tagging.Err != "" || report.Versioning.Err != "" { t.Fatalf("import: %+v", report) } meta, err = loadBucketMetadata(ctx, obj, bucket) diff --git a/cmd/site-replication.go b/cmd/site-replication.go index 41471fd63..b4807fd81 100644 --- a/cmd/site-replication.go +++ b/cmd/site-replication.go @@ -1411,9 +1411,12 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change var sp *policy.Policy var err error if len(change.Create.SessionPolicy) > 0 { - sp, err = policy.ParseConfig(bytes.NewReader(change.Create.SessionPolicy)) - if err != nil { - return iamReplicationError(err) + trimmedSP := bytes.TrimSpace(change.Create.SessionPolicy) + if len(trimmedSP) > 0 && !bytes.Equal(trimmedSP, []byte("null")) { + sp, err = parsePolicySnapshotBytes(trimmedSP) + if err != nil { + return iamReplicationError(err) + } } } // skip overwrite of local update if peer sent stale info @@ -1423,14 +1426,15 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change } } opts := newServiceAccountOpts{ - accessKey: change.Create.AccessKey, - secretKey: change.Create.SecretKey, - status: change.Create.Status, - sessionPolicy: sp, - claims: change.Create.Claims, - name: change.Create.Name, - description: change.Create.Description, - expiration: change.Create.Expiration, + accessKey: change.Create.AccessKey, + secretKey: change.Create.SecretKey, + status: change.Create.Status, + sessionPolicy: sp, + claims: change.Create.Claims, + name: change.Create.Name, + description: change.Create.Description, + expiration: change.Create.Expiration, + preserveEmptyEmbeddedPolicy: svcAcctClaimsHaveEmbeddedPolicy(change.Create.Claims), } _, _, err = globalIAMSys.NewServiceAccount(ctx, change.Create.Parent, change.Create.Groups, opts) if err != nil { @@ -1441,9 +1445,12 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change var sp *policy.Policy var err error if len(change.Update.SessionPolicy) > 0 { - sp, err = policy.ParseConfig(bytes.NewReader(change.Update.SessionPolicy)) - if err != nil { - return iamReplicationError(err) + trimmedSP := bytes.TrimSpace(change.Update.SessionPolicy) + if len(trimmedSP) > 0 && !bytes.Equal(trimmedSP, []byte("null")) { + sp, err = parsePolicySnapshotBytes(trimmedSP) + if err != nil { + return iamReplicationError(err) + } } } // skip overwrite of local update if peer sent stale info diff --git a/cmd/sse-kms-http-context_test.go b/cmd/sse-kms-http-context_test.go new file mode 100644 index 000000000..2e57a0b5e --- /dev/null +++ b/cmd/sse-kms-http-context_test.go @@ -0,0 +1,173 @@ +// Copyright (c) 2026 Feng Ruohang +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "encoding/base64" + "encoding/xml" + "maps" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/crypto" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/kms" +) + +// A successful encrypted write must stay readable. Malformed context must be +// rejected before replacing an existing object or creating a multipart upload. +func TestAPISSEKMSHTTPContext(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + // Specific PUT routes must precede the generic PutObject route. + endpoints: []string{"CopyObject", "NewMultipart", "PutObjectPart", "CompleteMultipart", "PutObject", "GetObject"}, + objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) { + previousKMS, previousTLS := GlobalKMS, globalIsTLS + var err error + GlobalKMS, err = kms.NewBuiltin("context-key", bytes.Repeat([]byte{0x42}, 32)) + if err != nil { + t.Fatal(err) + } + globalIsTLS = true + defer func() { GlobalKMS, globalIsTLS = previousKMS, previousTLS }() + request := func(t *testing.T, method, target string, body []byte, headers map[string]string) *httptest.ResponseRecorder { + t.Helper() + req, err := newTestSignedRequestV4(method, target, int64(len(body)), bytes.NewReader(body), cred.AccessKey, cred.SecretKey, headers) + if err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + return rec + } + expectStatus := func(t *testing.T, rec *httptest.ResponseRecorder, want int) { + t.Helper() + if rec.Code != want { + t.Fatalf("status=%d, want %d: %s", rec.Code, want, rec.Body.String()) + } + if want == http.StatusBadRequest { + var response APIErrorResponse + if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil || response.Code != "InvalidArgument" { + t.Fatalf("expected InvalidArgument: %s (decode: %v)", rec.Body.String(), err) + } + } + } + encryption := func(raw string) map[string]string { + return map[string]string{ + xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionKMS, + xhttp.AmzServerSideEncryptionKmsID: "context-key", + xhttp.AmzServerSideEncryptionKmsContext: base64.StdEncoding.EncodeToString([]byte(raw)), + } + } + for _, operation := range []string{"put", "self-copy", "copy-replace", "cross-key-copy", "multipart"} { + for _, tc := range []struct { + name, raw string + invalid bool + }{ + {"unicode", `{"区域":"杭州","emoji":"🔑","�":"replacement"}`, false}, + {"invalid-key", "{\"\xff\":\"value\"}", true}, + {"invalid-value", "{\"tenant\":\"\xff\"}", true}, + {"colliding-keys", "{\"\xff\":\"a\",\"\xfe\":\"b\"}", true}, + } { + t.Run(backend+"/"+operation+"/"+tc.name, func(t *testing.T) { + key := operation + "-" + tc.name + original, replacement := []byte("original encrypted object"), []byte("updated encrypted object") + seed := request(t, http.MethodPut, getPutObjectURL("", bucket, key), original, encryption(`{"tenant":"original"}`)) + expectStatus(t, seed, http.StatusOK) + before, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + headers := encryption(tc.raw) + wantData := replacement + var rec *httptest.ResponseRecorder + switch operation { + case "put": + rec = request(t, http.MethodPut, getPutObjectURL("", bucket, key), replacement, headers) + case "self-copy", "copy-replace", "cross-key-copy": + source := key + if operation == "cross-key-copy" { + source += "-source" + seed = request(t, http.MethodPut, getPutObjectURL("", bucket, source), replacement, encryption(`{"tenant":"source"}`)) + expectStatus(t, seed, http.StatusOK) + } else { + wantData = original + } + if operation == "copy-replace" { + headers[xhttp.AmzMetadataDirective] = "REPLACE" + } + headers[xhttp.AmzCopySource] = "/" + bucket + "/" + source + rec = request(t, http.MethodPut, getCopyObjectURL("", bucket, key), nil, headers) + case "multipart": + rec = request(t, http.MethodPost, getNewMultipartURL("", bucket, key), nil, headers) + if !tc.invalid { + expectStatus(t, rec, http.StatusOK) + var upload InitiateMultipartUploadResponse + if err := xml.Unmarshal(rec.Body.Bytes(), &upload); err != nil || upload.UploadID == "" { + t.Fatalf("missing upload ID: %s (decode: %v)", rec.Body.String(), err) + } + part := request(t, http.MethodPut, getPutObjectPartURL("", bucket, key, upload.UploadID, "1"), replacement, nil) + expectStatus(t, part, http.StatusOK) + // The in-process recorder retains the handler's literal + // ETag key; Header.Get would canonicalize it to Etag. + etags := part.Header()[xhttp.ETag] + if len(etags) != 1 || etags[0] == "" { + t.Fatalf("missing part ETag: %v", part.Header()) + } + body, err := xml.Marshal(CompleteMultipartUpload{Parts: []CompletePart{{PartNumber: 1, ETag: strings.Trim(etags[0], "\"")}}}) + if err != nil { + t.Fatal(err) + } + rec = request(t, http.MethodPost, getCompleteMultipartUploadURL("", bucket, key, upload.UploadID), body, nil) + } + } + wantStatus := http.StatusOK + if tc.invalid { + wantStatus, wantData = http.StatusBadRequest, original + } + expectStatus(t, rec, wantStatus) + get := request(t, http.MethodGet, getGetObjectURL("", bucket, key), nil, nil) + expectStatus(t, get, http.StatusOK) + if !bytes.Equal(get.Body.Bytes(), wantData) { + t.Fatalf("GET returned %q, want %q", get.Body.Bytes(), wantData) + } + after, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + if tc.invalid { + if !maps.Equal(before.UserDefined, after.UserDefined) || !before.ModTime.Equal(after.ModTime) { + t.Fatal("rejected context changed existing object metadata") + } + } else { + _, _, _, got, err := crypto.S3KMS.ParseMetadata(after.UserDefined) + want := kms.Context{"区域": "杭州", "emoji": "🔑", "�": "replacement"} + if err != nil || !maps.Equal(got, want) { + t.Fatalf("stored context=%v error=%v, want %v", got, err, want) + } + } + }) + } + } + }, + }) +} diff --git a/cmd/streaming-signature-v4.go b/cmd/streaming-signature-v4.go index eef479a0c..cdbcf40c3 100644 --- a/cmd/streaming-signature-v4.go +++ b/cmd/streaming-signature-v4.go @@ -153,6 +153,9 @@ func calculateSeedSignature(r *http.Request, trailers bool) (cred auth.Credentia if err != nil { return cred, "", "", time.Time{}, ErrMalformedDate } + if signV4Values.Credential.scope.date.Format(yyyymmdd) != date.UTC().Format(yyyymmdd) { + return cred, "", "", time.Time{}, ErrSignatureDoesNotMatch + } // Query string. queryStr := req.Form.Encode() @@ -444,6 +447,8 @@ func (cr *s3ChunkedReader) Read(buf []byte) (n int, err error) { return n, err } +const maxTrailerSize = 64 * 1024 + // readTrailers will read all trailers and populate cr.trailers with actual values. func (cr *s3ChunkedReader) readTrailers() error { if cr.debug { @@ -457,8 +462,12 @@ func (cr *s3ChunkedReader) readTrailers() error { if err == io.EOF { return io.ErrUnexpectedEOF } + return err } if v != '\r' { + if valueBuffer.Len() >= maxTrailerSize { + return errMalformedEncoding + } valueBuffer.WriteByte(v) continue } @@ -468,6 +477,7 @@ func (cr *s3ChunkedReader) readTrailers() error { if err == io.EOF { return io.ErrUnexpectedEOF } + return err } if v != '\n' { return errMalformedEncoding @@ -483,8 +493,12 @@ func (cr *s3ChunkedReader) readTrailers() error { if err == io.EOF { return io.ErrUnexpectedEOF } + return err } if v != '\r' { + if signatureBuffer.Len() >= maxTrailerSize { + return errMalformedEncoding + } signatureBuffer.WriteByte(v) continue } @@ -494,6 +508,7 @@ func (cr *s3ChunkedReader) readTrailers() error { if err == io.EOF { return io.ErrUnexpectedEOF } + return err } if string(tmp[:]) != "\n\r\n" { if cr.debug { diff --git a/cmd/streaming-v4-unsigned.go b/cmd/streaming-v4-unsigned.go index 11c524a3a..e62af8fe2 100644 --- a/cmd/streaming-v4-unsigned.go +++ b/cmd/streaming-v4-unsigned.go @@ -217,8 +217,12 @@ func (cr *s3UnsignedChunkedReader) readTrailers() error { if err == io.EOF { return io.ErrUnexpectedEOF } + return err } if v != '\r' { + if valueBuffer.Len() >= maxTrailerSize { + return errMalformedEncoding + } valueBuffer.WriteByte(v) continue } @@ -229,6 +233,7 @@ func (cr *s3UnsignedChunkedReader) readTrailers() error { if err == io.EOF { return io.ErrUnexpectedEOF } + return err } if !bytes.Equal(tmp[:], []byte{'\n', '\r', '\n'}) { if cr.debug { diff --git a/cmd/sts-handlers.go b/cmd/sts-handlers.go index 4ffc649c7..41d236616 100644 --- a/cmd/sts-handlers.go +++ b/cmd/sts-handlers.go @@ -18,7 +18,6 @@ package cmd import ( - "bytes" "context" "crypto/x509" "encoding/base64" @@ -120,7 +119,7 @@ func (c stsClaims) populateSessionPolicy(form url.Values) error { return nil } - sessionPolicy, err := policy.ParseConfig(bytes.NewReader([]byte(sessionPolicyStr))) + sessionPolicy, err := parseSessionPolicyStrictBytes([]byte(sessionPolicyStr)) if err != nil { return err } diff --git a/cmd/test-utils_test.go b/cmd/test-utils_test.go index 54701298a..fd7209cba 100644 --- a/cmd/test-utils_test.go +++ b/cmd/test-utils_test.go @@ -62,6 +62,7 @@ import ( "github.com/minio/minio-go/v7/pkg/signer" "github.com/minio/minio/internal/auth" "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/config/storageclass" "github.com/minio/minio/internal/crypto" "github.com/minio/minio/internal/hash" "github.com/minio/minio/internal/logger" @@ -188,6 +189,10 @@ func calculateStreamContentLength(dataLen, chunkSize int64) int64 { } func prepareFS(ctx context.Context) (ObjectLayer, string, error) { + return prepareFSWithDisks(ctx, nil) +} + +func prepareFSWithDisks(ctx context.Context, wrapDisk func(StorageAPI) StorageAPI) (ObjectLayer, string, error) { nDisks := 1 fsDirs, err := getRandomDisks(nDisks) if err != nil { @@ -198,12 +203,35 @@ func prepareFS(ctx context.Context) (ObjectLayer, string, error) { return nil, "", err } + wrapTestDisks(obj, wrapDisk) initAllSubsystems(ctx) globalIAMSys.Init(ctx, obj, globalEtcdClient, 2*time.Second) return obj, fsDirs[0], nil } +// wrapTestDisks installs fixture disks before IAM starts reading them. Keep +// the production getters unchanged and retain the wrappers until shutdown; +// restoring a getter while an IAM Walk is in flight races that reader. +func wrapTestDisks(obj ObjectLayer, wrapDisk func(StorageAPI) StorageAPI) { + if wrapDisk == nil { + return + } + for _, pool := range obj.(*erasureServerPools).serverPools { + pool.erasureDisksMu.Lock() + for i, set := range pool.erasureDisks { + disks := append([]StorageAPI(nil), set...) + for j, disk := range disks { + if disk != nil { + disks[j] = wrapDisk(disk) + } + } + pool.erasureDisks[i] = disks + } + pool.erasureDisksMu.Unlock() + } +} + func prepareErasureSets32(ctx context.Context) (ObjectLayer, []string, error) { return prepareErasure(ctx, 32) } @@ -505,6 +533,9 @@ func resetTestGlobals() { resetGlobalIAMSys() // Object-layer fixtures must not reuse the preceding IAM suite's etcd store. globalEtcdClient = nil + storageclass.ConfigLock.Lock() + globalStorageClass = storageclass.Config{} + storageclass.ConfigLock.Unlock() } // Configure the server for the test run. @@ -1752,6 +1783,7 @@ type ExecObjectLayerAPITestArgs struct { objAPITest objAPITestType endpoints []string init func() + wrapDisk func(StorageAPI) StorageAPI makeBucketOptions MakeBucketOptions } @@ -1765,7 +1797,7 @@ func ExecObjectLayerAPITest(args ExecObjectLayerAPITestArgs) { // this is to make sure that the tests are not affected by modified value. resetTestGlobals() - objLayer, fsDir, err := prepareFS(ctx) + objLayer, fsDir, err := prepareFSWithDisks(ctx, args.wrapDisk) if err != nil { args.t.Fatalf("Initialization of object layer failed for single node setup: %s", err) } @@ -1800,6 +1832,7 @@ func ExecObjectLayerAPITest(args ExecObjectLayerAPITestArgs) { } defer objLayer.Shutdown(ctx) + wrapTestDisks(objLayer, args.wrapDisk) bucketErasure, erAPIRouter, err := initAPIHandlerTest(ctx, objLayer, args.endpoints, args.makeBucketOptions) if err != nil { args.t.Fatalf("Initialization of API handler tests failed: %s", err) diff --git a/go.mod b/go.mod index e4fa95340..14061c907 100644 --- a/go.mod +++ b/go.mod @@ -1,17 +1,12 @@ module github.com/minio/minio -go 1.27.3 +go 1.27.4 // Console and MC retain their historical module paths for best-effort upstream // compatibility. Pin the maintained PGSTY implementations used by SILO. -replace github.com/minio/console => github.com/pgsty/silo-console v0.0.0-20261109114619-4e91ae506e70 +replace github.com/minio/console => github.com/pgsty/silo-console v0.0.0-20261111181657-3825ebc44451 -replace github.com/minio/mc => github.com/pgsty/mc v0.0.0-20261109021353-e1d4cc5abfe0 - -// v22.7.0 does not compile on NetBSD because its unix implementation uses -// CLOCK_MONOTONIC, which is unavailable there. Keep the last portable release -// until go-systemd ships the upstream fix. -replace github.com/coreos/go-systemd/v22 => github.com/coreos/go-systemd/v22 v22.6.0 +replace github.com/minio/mc => github.com/pgsty/mc v0.0.0-20261111180825-a37a9f054fba // Install tools using 'go install tool'. tool ( @@ -24,7 +19,7 @@ require ( cloud.google.com/go/storage v1.70.0 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.3 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.15.0 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.3 github.com/IBM/sarama v1.61.1 github.com/alecthomas/participle v0.7.1 github.com/beevik/ntp v1.6.0 @@ -35,7 +30,7 @@ require ( github.com/coreos/go-systemd/v22 v22.7.0 github.com/cosnicolaou/pbzip2 v1.0.6 github.com/dchest/siphash v1.2.3 - github.com/dustin/go-humanize v1.0.1 + github.com/dustin/go-humanize v1.1.0 github.com/eclipse/paho.mqtt.golang v1.5.1 github.com/elastic/go-elasticsearch/v7 v7.17.10 github.com/fatih/color v1.19.0 @@ -50,7 +45,7 @@ require ( github.com/google/uuid v1.6.0 github.com/inconshreveable/mousetrap v1.1.0 github.com/json-iterator/go v1.1.12 - github.com/klauspost/compress v1.20.3 + github.com/klauspost/compress v1.20.4 github.com/klauspost/cpuid/v2 v2.4.2 github.com/klauspost/filepathx v1.1.1 github.com/klauspost/pgzip v1.3.0 @@ -81,7 +76,7 @@ require ( github.com/nats-io/stan.go v0.10.4 github.com/ncw/directio v1.0.5 github.com/nsqio/go-nsq v1.1.0 - github.com/pgsty/silo-pkg/v3 v3.16.0 + github.com/pgsty/silo-pkg/v3 v3.16.1 github.com/philhofer/fwd v1.2.1 github.com/pierrec/lz4/v4 v4.1.33 github.com/pkg/errors v0.9.1 @@ -108,10 +103,11 @@ require ( go.yaml.in/yaml/v3 v3.0.6 goftp.io/server/v2 v2.0.3 golang.org/x/crypto v0.58.1-0.20261009181718-2f1f834330b2 - golang.org/x/oauth2 v0.37.0 - golang.org/x/sync v0.24.0 - golang.org/x/sys v0.49.0 - golang.org/x/term v0.47.0 + golang.org/x/mod v0.43.0 + golang.org/x/oauth2 v0.38.0 + golang.org/x/sync v0.25.0 + golang.org/x/sys v0.50.0 + golang.org/x/term v0.48.0 golang.org/x/time v0.16.0 google.golang.org/api v0.308.0 gopkg.in/yaml.v2 v2.4.0 @@ -122,11 +118,11 @@ require ( aead.dev/minisign v0.3.0 // indirect cel.dev/expr v0.25.3 // indirect cloud.google.com/go v0.123.0 // indirect - cloud.google.com/go/auth v0.24.3 // indirect + cloud.google.com/go/auth v0.24.4 // indirect cloud.google.com/go/auth/oauth2adapt v0.3.1 // indirect cloud.google.com/go/compute/metadata v0.10.0 // indirect - cloud.google.com/go/iam v1.15.0 // indirect - cloud.google.com/go/monitoring v1.32.0 // indirect + cloud.google.com/go/iam v1.15.1 // indirect + cloud.google.com/go/monitoring v1.32.1 // indirect filippo.io/edwards25519 v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.13.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0 // indirect @@ -201,18 +197,18 @@ require ( github.com/google/pprof v0.0.0-20261016150608-1facf8f91b50 // indirect github.com/google/s2a-go v0.1.11 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.24 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.25 // indirect github.com/googleapis/gax-go/v2 v2.27.0 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.32.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.33.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-uuid v1.0.4 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.11.0 // indirect - github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/jackc/pgx/v5 v5.12.0 // indirect + github.com/jackc/puddle/v2 v2.2.3 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/gofork v1.7.6 // indirect @@ -279,12 +275,12 @@ require ( github.com/tidwall/pretty v1.2.2 // indirect github.com/tklauser/go-sysconf v0.4.0 // indirect github.com/tklauser/numcpus v0.12.0 // indirect - github.com/unrolled/secure v1.17.0 // indirect + github.com/unrolled/secure v1.18.0 // indirect github.com/valyala/fastjson v1.6.10 // indirect github.com/vbauerster/cupwriter v0.0.5 // indirect github.com/vbauerster/mpb/v8 v8.16.3 // indirect github.com/xdg/stringprep v1.0.3 // indirect - github.com/xo/terminfo v1.2.0 // indirect + github.com/xo/terminfo v1.2.1 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/etcd/client/pkg/v3 v3.7.3 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect @@ -299,14 +295,13 @@ require ( go.opentelemetry.io/otel/trace v1.47.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20261014143641-f0251eb4aa3e // indirect - golang.org/x/mod v0.42.0 // indirect golang.org/x/net v0.62.0 // indirect - golang.org/x/text v0.43.0 // indirect + golang.org/x/text v0.44.0 // indirect golang.org/x/tools v0.52.0 // indirect google.golang.org/genproto v0.0.0-20261019180522-ce908d29299b // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20261019180522-ce908d29299b // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20261026165753-2ffab32bce7e // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20261019180522-ce908d29299b // indirect - google.golang.org/grpc v1.84.0 // indirect + google.golang.org/grpc v1.85.0 // indirect google.golang.org/protobuf v1.36.12 // indirect gopkg.in/ini.v1 v1.67.3 // indirect ) diff --git a/go.sum b/go.sum index eabd2d4bd..cef3e6f01 100644 --- a/go.sum +++ b/go.sum @@ -9,20 +9,20 @@ cel.dev/expr v0.25.3 h1:A2jO8jwOugrrovveCWfj0KEZOfqiLgAcwjpHPhzIGw0= cel.dev/expr v0.25.3/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= -cloud.google.com/go/auth v0.24.3 h1:oVmRvR/pGSCNXjviwXCisJjRSEdQ0BlcOpW0uhuHLm8= -cloud.google.com/go/auth v0.24.3/go.mod h1:p4UXqfSXnMVUDutk9xePzI8lj8yyoJ3BLsKWv0NSvQk= +cloud.google.com/go/auth v0.24.4 h1:m49juFKXOaeRwmLgcrd3e4hrwqLDe1+4g8fH0EoznUU= +cloud.google.com/go/auth v0.24.4/go.mod h1:p4UXqfSXnMVUDutk9xePzI8lj8yyoJ3BLsKWv0NSvQk= cloud.google.com/go/auth/oauth2adapt v0.3.1 h1:BKLDdbWS7pafd820dsyFDF6fPEzIGqr7ny6hbTzCCgE= cloud.google.com/go/auth/oauth2adapt v0.3.1/go.mod h1:B1nM4FJDxoDqAt3PXFG4IflUoQ5kvWAOOM3R+5wAszg= cloud.google.com/go/compute/metadata v0.10.0 h1:pyKMUQSwchgkIBBJGdILqQbs/BNJXqwSA7Ej6LAvvtY= cloud.google.com/go/compute/metadata v0.10.0/go.mod h1:rGFHRrIif570kSibjFTMbt6/4/tzgJWFGI/HVol4GIk= -cloud.google.com/go/iam v1.15.0 h1:U/wqjfrM0ceBFdPlOI2+cu68yPAxh72M8fe9uLG0+ZE= -cloud.google.com/go/iam v1.15.0/go.mod h1:HAVf7PNhTrNTrEc6fVB21l8ejkbzNTulJHYf7QtD424= +cloud.google.com/go/iam v1.15.1 h1:i6EXLKyIi0Sh72p+5+Fz0PfnzB63bkneVpaBCGrjlWs= +cloud.google.com/go/iam v1.15.1/go.mod h1:nDJ/rfCzLKyBUY1AeiSQwOAJg/AN7zZ2QmNzRTcK2jM= cloud.google.com/go/logging v1.20.0 h1:l7yzCB8/JSY5PHT9RsaqdT1jaBGbM6KlSJDX210HlXM= cloud.google.com/go/logging v1.20.0/go.mod h1:ALUaJQZM691Eeujy9kFx0xREq3AMPuGd+pTRZuTYv+w= cloud.google.com/go/longrunning v1.3.0 h1:lZ6c8SJJBmILp/Wnr3vGnBLa0q+cYfSHRCcUfVI2hpI= cloud.google.com/go/longrunning v1.3.0/go.mod h1:wpgLxCn+UA3/TZ1bj/fW39mfDpwVlMPu1xKJyoMt7V0= -cloud.google.com/go/monitoring v1.32.0 h1:0bXxoh3o751jZnKDHgUDajaG8ZtqtKPMxhVmohRHI0Q= -cloud.google.com/go/monitoring v1.32.0/go.mod h1:FxmFKr7SUhb60LDuYvjUc6bk9i1+LyXaENR5qAEibrM= +cloud.google.com/go/monitoring v1.32.1 h1:DzNqsbCCyMJ+Oq+McBHaR5qqqxCvdNCIgfJug/YM0Ys= +cloud.google.com/go/monitoring v1.32.1/go.mod h1:JkJhVXtTcGh6KQuvProJ9olvu2p37Ay29CinCDFKHHg= cloud.google.com/go/storage v1.70.0 h1:LQNHn0rRF94jVFfF3e416IzdQqIufa5GfF6HilQxb80= cloud.google.com/go/storage v1.70.0/go.mod h1:W84cNa070TZIegvTW67CBY+f5rxux8EJMb7YyiHdrls= cloud.google.com/go/trace v1.16.0 h1:GmQovzFc5F0CNfl0VLgL64aoTtu7xsM0YajW2GlG9+E= @@ -39,8 +39,8 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.13.0 h1:Terj1CMW4+NY5CpWjSRkqL github.com/Azure/azure-sdk-for-go/sdk/internal v1.13.0/go.mod h1:7Ut2Qw9vcwpLW5Q4MBy43uSxXS7tK2yjs94OJ6j5thE= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2 h1:tQJdgQjbKKffEiMOKJmcQPkY01eFHRqWIZVefqP44nI= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2/go.mod h1:kTkpJchXAdO7LGkKZP97a+Ou8TgaF4NT6MHwOIy5uSQ= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.3 h1:zoPQ7KXGgzqeUmNWl+b6IwVAqsb5lROEJhqAmqXQtm0= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.3/go.mod h1:ei/sDx/dwDIUtZE3PQP1u8+qrA+CRy4q7qW+A2MbgB4= github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0 h1:X/bTwe0qpvmvPPutc+LWo7T4cKMqDP9JHP4k+Co9ZB4= github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0/go.mod h1:+kX7ahyFelQ42F3RScM2CFgHJljp0phc/FieKLYVpFI= github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDPw= @@ -132,8 +132,8 @@ github.com/coreos/go-oidc/v3 v3.21.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= -github.com/coreos/go-systemd/v22 v22.6.0 h1:aGVa/v8B7hpb0TKl0MWoAavPDmHvobFe5R5zn0bCJWo= -github.com/coreos/go-systemd/v22 v22.6.0/go.mod h1:iG+pp635Fo7ZmV/j14KUcmEyWF+0X7Lua8rrTWzYgWU= +github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= +github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= github.com/cosnicolaou/pbzip2 v1.0.6 h1:FYF6b2j4X4q3hZezd2AoUN/emLCtH/MbDGwJjiOacak= github.com/cosnicolaou/pbzip2 v1.0.6/go.mod h1:uCNfm0iE2wIKGRlLyq31M4toziFprNhEnvueGmh5u3M= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= @@ -147,8 +147,8 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3h github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg= +github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI= github.com/eapache/go-resiliency v1.7.0 h1:n3NRTnBn5N0Cbi/IeOHuQn9s2UwVUH7Ga0ZWcP+9JTA= github.com/eapache/go-resiliency v1.7.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/eclipse/paho.mqtt.golang v1.5.1 h1:/VSOv3oDLlpqR2Epjn1Q7b2bSTplJIeV2ISgCl2W7nE= @@ -288,8 +288,8 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaU github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.24 h1:G9iCJIY+AYVmUskKdjX8FVy7sG+nbzjOQ41cvlmhcvg= -github.com/googleapis/enterprise-certificate-proxy v0.3.24/go.mod h1:hEjCmnoakQzv5ImLUp6alkY7XgLlNoAxwTXmhHHNIsI= +github.com/googleapis/enterprise-certificate-proxy v0.3.25 h1:n/6K8rCcoQQjGZ+o5irYrdwAFZIclQMXq0yeJyK2FaI= +github.com/googleapis/enterprise-certificate-proxy v0.3.25/go.mod h1:hEjCmnoakQzv5ImLUp6alkY7XgLlNoAxwTXmhHHNIsI= github.com/googleapis/gax-go/v2 v2.27.0 h1:qe2iQOUzxb3fd6x6ewpIEixPQhJ+0Ihc/RRZhKA/2g0= github.com/googleapis/gax-go/v2 v2.27.0/go.mod h1:RofiqDw3ehcG4L0H1zO3D2Itpwd3/uS/7wdmbNDF1DE= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= @@ -298,8 +298,8 @@ github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5T github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 h1:cLN4IBkmkYZNnk7EAJ0BHIethd+J6LqxFNw5mSiI2bM= github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.32.0 h1:QMILzF8BA7xftE4zVORkRrlv08m12St3csK4RsyRgRs= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.32.0/go.mod h1:+ZQ6iUL8+ATyxGDQNuRKpNwJFdGDUi0e7E1QL9MvI7U= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.33.0 h1:5T5Ns3kMpWr+z3s5ZA0e7ljVyWNVNxK76sHPAxi5D1k= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.33.0/go.mod h1:oWNYw09cH8wl8fFgC5hKlBLxR4XfKy2KQpP8MJjeCcc= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -329,10 +329,10 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= -github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= -github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= -github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jackc/pgx/v5 v5.12.0 h1:hWwax8YiGT0YiN50U1oIDn5kgR7KC2tG3rmavwpPkxY= +github.com/jackc/pgx/v5 v5.12.0/go.mod h1:u2Fi6no03HCSBYE+ZVdLxbwdu3xyi2vU+K/YNJioSds= +github.com/jackc/puddle/v2 v2.2.3 h1:f8HoREISb3l5eO9coPgK7U3hKLhWGokOFVcThl5jrao= +github.com/jackc/puddle/v2 v2.2.3/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -358,8 +358,8 @@ github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRt github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4= -github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/klauspost/compress v1.20.4 h1:dD0v/bvxtjCBqQu6sx1TNWkalEY5lNTIe+HY9w8TeNI= +github.com/klauspost/compress v1.20.4/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.4.2 h1:xRw7IZEsAgVCp6trS/DgWbTAqpixhmywgVgBjYGg1bQ= github.com/klauspost/cpuid/v2 v2.4.2/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= @@ -514,12 +514,12 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6 github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pgsty/mc v0.0.0-20261109021353-e1d4cc5abfe0 h1:dWOBJcDpTM2dWOnO4aQXOOxjTWFbjYZJY+2TuCHTI2c= -github.com/pgsty/mc v0.0.0-20261109021353-e1d4cc5abfe0/go.mod h1:OgTnkfk6tqIYVtAlU+aB4kuOXXkG8S3A0XSOSJQ2VgQ= -github.com/pgsty/silo-console v0.0.0-20261109114619-4e91ae506e70 h1:LAl1Bssw+N3L1DPH/WHS+6DseeAlhzN6hXZ50jpS4zI= -github.com/pgsty/silo-console v0.0.0-20261109114619-4e91ae506e70/go.mod h1:q+v/BpLadZ4aYVmRTYuK4tm+AYqtbE6gNwVG+joMlx4= -github.com/pgsty/silo-pkg/v3 v3.16.0 h1:k8g+jpZy90TLtuev9S5g7iTD7Kuw3zg1RNVaBT+UqbQ= -github.com/pgsty/silo-pkg/v3 v3.16.0/go.mod h1:V2hZewi0YPHkGKmFkB0CMC6Gfr9dD6F7IomaJxxdw3c= +github.com/pgsty/mc v0.0.0-20261111180825-a37a9f054fba h1:KKLNRed2r86tWb/dqcjaAY7wiP6VmOaXe137l+Y/tLU= +github.com/pgsty/mc v0.0.0-20261111180825-a37a9f054fba/go.mod h1:M/s9Mioaya7h38sTw65xqyhfDYB5L3aTkgjAr1e7ZAo= +github.com/pgsty/silo-console v0.0.0-20261111181657-3825ebc44451 h1:TRndCfb0fpRqg5Zw9qkZHCATBdet5USpBcjLCBb1GJw= +github.com/pgsty/silo-console v0.0.0-20261111181657-3825ebc44451/go.mod h1:6To0hfiYfPNTu5O0p3VpYFn6Z3od9y6YLE5wTR+TXWs= +github.com/pgsty/silo-pkg/v3 v3.16.1 h1:jk8clnx7Fg5ZcpdzqP137XQ4SSAQDF/Kaa/12k721Os= +github.com/pgsty/silo-pkg/v3 v3.16.1/go.mod h1:ktr9Y2OO6QAQU+9bkrS7GGqUwM9AQR5jITitySmMSEg= github.com/philhofer/fwd v1.2.1 h1:UtpS73q8bFgy/yH3gCq8LSNvj5gv/FowNOiWZa7uKwQ= github.com/philhofer/fwd v1.2.1/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pierrec/lz4/v4 v4.1.33 h1:GjG1TJ1V4IzKP8L96muuuDNpTwd7D+l2ccXrjAbe014= @@ -619,8 +619,8 @@ github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zu github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/unrolled/secure v1.17.0 h1:Io7ifFgo99Bnh0J7+Q+qcMzWM6kaDPCA5FroFZEdbWU= -github.com/unrolled/secure v1.17.0/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40= +github.com/unrolled/secure v1.18.0 h1:NfL/H0WI3KLnW9fp6OuTSQ7hHArfnRri/9hddUT9Acs= +github.com/unrolled/secure v1.18.0/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fastjson v1.6.10 h1:/yjJg8jaVQdYR3arGxPE2X5z89xrlhS0eGXdv+ADTh4= @@ -633,8 +633,8 @@ github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw= github.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= -github.com/xo/terminfo v1.2.0 h1:d0ZTOCpuGE0lwSAOs0zcJjwz3jWQyqcRt9XbGJpCOl4= -github.com/xo/terminfo v1.2.0/go.mod h1:lGzkSo8Fe7IRh/w+Gqz7n5mDog4FVXCj3gy/DYTBqio= +github.com/xo/terminfo v1.2.1 h1:e5fQoon4xY/NK3LJzcF8me+JaknbLD8zjtRWm0uvxtw= +github.com/xo/terminfo v1.2.1/go.mod h1:lGzkSo8Fe7IRh/w+Gqz7n5mDog4FVXCj3gy/DYTBqio= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -709,8 +709,8 @@ golang.org/x/exp v0.0.0-20261014143641-f0251eb4aa3e/go.mod h1:DBoxwWhjaW8DJJw/Ye golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg= -golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs= +golang.org/x/mod v0.43.0 h1:MFuq5J6s35W9D3IjkwyWc9vHh0nENOrt5K8Iof4ofmg= +golang.org/x/mod v0.43.0/go.mod h1:4ec1JnuyBz5+FIVhg86UsA39YG8BpnrZdJYSNHmGzHY= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -724,15 +724,15 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= -golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= -golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= +golang.org/x/oauth2 v0.38.0 h1:uW7XQbcu1eoiQS7uMJvLOjPMl04bTbmY1VeRPxvbspI= +golang.org/x/oauth2 v0.38.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= -golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sync v0.25.0 h1:dJnifr0nMV8/SBnBvdo4eSGGnrc3O4FFx6IqvtLcmRs= +golang.org/x/sync v0.25.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -757,23 +757,23 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= -golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/sys v0.50.0 h1:dkKZEFTEKaLr1sjhs0WiKqoi8Jj5nHbS3Njmx1UoMJs= +golang.org/x/sys v0.50.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM= -golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY= +golang.org/x/term v0.48.0 h1:G6UU1eiSB8nl3DzeUGt7Mg+2SctI8tPS6Z5/mBt6cOk= +golang.org/x/term v0.48.0/go.mod h1:BRDLHCZr2SUjIeBHHpRyQeC9xzSAVbzY2rvKgafnHf4= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= -golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= +golang.org/x/text v0.44.0 h1:YItdp/mQNGnsmrgpcdA5yQ3bnRxCfo/Tj/iByGW9GGo= +golang.org/x/text v0.44.0/go.mod h1:S4FR1waQ9LbjQTls97SyRjDaUUVkaM+cXwwuG/G+MwY= golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE= golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -793,12 +793,12 @@ google.golang.org/api v0.308.0 h1:iGb+4qT3nVb3uuIzwor3yP2nyu+rMZr31PvLFiRNJQk= google.golang.org/api v0.308.0/go.mod h1:4XMH+b/ZEfuTbK7FnrFRY9c4xfFHVflGTKuy/i5wUmE= google.golang.org/genproto v0.0.0-20261019180522-ce908d29299b h1:+t2qGqt6R8CO622uiX1JpnRvRvSOQoVRV0DiPzUjKXg= google.golang.org/genproto v0.0.0-20261019180522-ce908d29299b/go.mod h1:XRIfc1yr5xsbF8xFw5+g7SEmLJX/eQ7MMazTRbnMMvI= -google.golang.org/genproto/googleapis/api v0.0.0-20261019180522-ce908d29299b h1:nB3spRIaSPtxJpxb0k41yljROvm2Jrwfs2pSNdKS5y8= -google.golang.org/genproto/googleapis/api v0.0.0-20261019180522-ce908d29299b/go.mod h1:9N4vtjEwvgQKEdlFVQO0M44ErhecQ9DCSQfGoZ51M6w= +google.golang.org/genproto/googleapis/api v0.0.0-20261026165753-2ffab32bce7e h1:e789sLZjeTdc02ZCQGTSzHOu0fR73+dwPQVKUp67Z88= +google.golang.org/genproto/googleapis/api v0.0.0-20261026165753-2ffab32bce7e/go.mod h1:vHV3wJM9WZyfV7GyB+OQGq5dWVrFwCDxapPK4WlRD88= google.golang.org/genproto/googleapis/rpc v0.0.0-20261019180522-ce908d29299b h1:6WNWHZk91dp4gZ25642OALkuG1UVNZbwENJD19DnB8k= google.golang.org/genproto/googleapis/rpc v0.0.0-20261019180522-ce908d29299b/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= -google.golang.org/grpc v1.84.0 h1:soMyaPJ8pAak5PIQ0DGBUir0XRo2fRoMqhNWMLlLxO0= -google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0= +google.golang.org/grpc v1.85.0 h1:BPBCfuVRQ1qvBhKDOq9uvQllMvO59rKsYhXpp7Dd8vw= +google.golang.org/grpc v1.85.0/go.mod h1:0rQrOD58yjal/0EaVbGBANindixsCZNQPDV3VKtaKuQ= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/helm/silo/Chart.yaml b/helm/silo/Chart.yaml index 1daee4a44..9fee02110 100644 --- a/helm/silo/Chart.yaml +++ b/helm/silo/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: S3-Interface Libre Object Storage name: silo -version: 7.0.4 -appVersion: RELEASE.2026-11-09T00-00-00Z +version: 7.0.5 +appVersion: RELEASE.2026-11-12T00-00-00Z keywords: - silo - storage diff --git a/helm/silo/README.md b/helm/silo/README.md index 255fc81db..c2c45177b 100644 --- a/helm/silo/README.md +++ b/helm/silo/README.md @@ -101,18 +101,19 @@ new chart invokes `silo server`; an old image does not provide that executable. The new image accepts a legacy first argv token of `minio` only to support old chart commands during the forward migration. -When upgrading an existing Silo release to `RELEASE.2026-11-09T00-00-00Z` (chart -`7.0.4`): +When upgrading an existing Silo release to `RELEASE.2026-11-12T00-00-00Z` (chart +`7.0.5`): - `helm upgrade --reuse-values` keeps the previously saved `image.tag` and - `mcImage.tag`; pass `--set-string image.tag=RELEASE.2026-11-09T00-00-00Z` and - `--set-string mcImage.tag=RELEASE.2026-11-09T00-00-00Z` explicitly. -- Drain or cancel active batch jobs before upgrading; `20261026` cannot resume - new submitter-bound batch manifests on downgrade. -- Single-pool clusters whose erasure sets do not all share the same pods/hosts - require a drained stop-all upgrade rather than a rolling pod restart, and - `POST ?restore` returns `503` while mixed versions are serving. See the - [20261109 release notes](https://silo.pgsty.com/blog/release/silo-20261109/#upgrade). + `mcImage.tag`; pass `--set-string image.tag=RELEASE.2026-11-12T00-00-00Z` and + `--set-string mcImage.tag=RELEASE.2026-11-12T00-00-00Z` explicitly. +- When upgrading from `20261026` or earlier, drain or cancel active batch jobs + before upgrading; `20261026` cannot resume submitter-bound batch manifests on + downgrade, single-pool clusters whose erasure sets do not all share the same + pods/hosts require a drained stop-all upgrade rather than a rolling pod + restart, and `POST ?restore` returns `503` while mixed versions are serving. + See the [20261109 release notes](https://silo.pgsty.com/blog/release/silo-20261109/#upgrade) + and [server compatibility notes](https://silo.pgsty.com/compatibility/server/). ## Persistence and TLS diff --git a/helm/silo/values.yaml b/helm/silo/values.yaml index c7ee7ded4..52d8df3b1 100644 --- a/helm/silo/values.yaml +++ b/helm/silo/values.yaml @@ -15,7 +15,7 @@ clusterDomain: cluster.local ## image: repository: pgsty/silo - tag: RELEASE.2026-11-09T00-00-00Z + tag: RELEASE.2026-11-12T00-00-00Z pullPolicy: IfNotPresent imagePullSecrets: [] @@ -25,7 +25,7 @@ imagePullSecrets: [] ## mcImage: repository: pgsty/mc - tag: RELEASE.2026-11-09T00-00-00Z + tag: RELEASE.2026-11-12T00-00-00Z pullPolicy: IfNotPresent ## Silo mode, i.e. standalone or distributed. diff --git a/internal/crypto/sse-kms.go b/internal/crypto/sse-kms.go index dd0aa46a2..1a42c9606 100644 --- a/internal/crypto/sse-kms.go +++ b/internal/crypto/sse-kms.go @@ -24,6 +24,7 @@ import ( "net/http" "path" "strings" + "unicode/utf8" jsoniter "github.com/json-iterator/go" xhttp "github.com/minio/minio/internal/http" @@ -79,6 +80,15 @@ func (ssekms) ParseHTTP(h http.Header) (string, kms.Context, error) { if err != nil { return "", nil, err } + // Persisting invalid UTF-8 replaces bytes and can collapse distinct + // keys, changing the associated data needed to decrypt the object. + if !utf8.Valid(b) { + return "", nil, kms.Error{ + Code: http.StatusBadRequest, + APICode: "InvalidArgument", + Err: "KMS context must be valid UTF-8", + } + } json := jsoniter.ConfigCompatibleWithStandardLibrary if err := json.Unmarshal(b, &ctx); err != nil { diff --git a/internal/crypto/sse-kms_test.go b/internal/crypto/sse-kms_test.go new file mode 100644 index 000000000..69fff133c --- /dev/null +++ b/internal/crypto/sse-kms_test.go @@ -0,0 +1,66 @@ +// Copyright (c) 2026 Feng Ruohang +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package crypto + +import ( + "encoding/base64" + "errors" + "maps" + "net/http" + "testing" + + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/kms" +) + +func TestKMSParseHTTPContext(t *testing.T) { + for _, tc := range []struct { + name, raw string + want kms.Context + invalid bool + }{ + {name: "absent"}, + {name: "empty", raw: `{}`, want: kms.Context{}}, + {name: "null", raw: `null`}, + {name: "unicode", raw: `{"区域":"杭州","emoji":"🔑","�":"replacement"}`, want: kms.Context{"区域": "杭州", "emoji": "🔑", "�": "replacement"}}, + {name: "invalid-key", raw: "{\"\xff\":\"value\"}", invalid: true}, + {name: "invalid-value", raw: "{\"tenant\":\"\xff\"}", invalid: true}, + {name: "colliding-keys", raw: "{\"\xff\":\"a\",\"\xfe\":\"b\"}", invalid: true}, + {name: "replacement-key", raw: "{\"\xff\":\"a\",\"�\":\"b\"}", invalid: true}, + } { + t.Run(tc.name, func(t *testing.T) { + header := http.Header{} + header.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS) + header.Set(xhttp.AmzServerSideEncryptionKmsID, "context-key") + if tc.raw != "" { + header.Set(xhttp.AmzServerSideEncryptionKmsContext, base64.StdEncoding.EncodeToString([]byte(tc.raw))) + } + key, context, err := S3KMS.ParseHTTP(header) + if tc.invalid { + var apiErr kms.Error + if !errors.As(err, &apiErr) || apiErr.Code != http.StatusBadRequest || apiErr.APICode != "InvalidArgument" { + t.Fatalf("invalid UTF-8: got %v, want 400 InvalidArgument", err) + } + return + } + if err != nil || key != "context-key" || !maps.Equal(context, tc.want) { + t.Fatalf("key=%q context=%v error=%v; want context=%v", key, context, err, tc.want) + } + }) + } +}