mirror of
https://github.com/pgsty/minio.git
synced 2026-11-17 04:49:33 +03:00
Compare commits
52 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 9d49688c87 | |||
| 8e09374cb8 | |||
| 58d90ed73c | |||
| e857b6741d | |||
| 0505ef83b5 | |||
| 22bc15d89b | |||
| a2e904b966 | |||
| cc7dc61eb4 | |||
| c4f480a839 | |||
| 60831e3299 | |||
| 037319066f | |||
| bb871a7c31 | |||
| 0ebbd3caef | |||
| bd56f80250 | |||
| a39e810965 | |||
| 09103991ea | |||
| c43f745449 | |||
| 9610a74c19 | |||
| 0bcd8abc5c | |||
| 70b350c383 | |||
| 338e9a9be9 | |||
| edbd8709ec | |||
| 5db60a6c59 | |||
| 36c19f1d65 | |||
| d3f9f8be88 | |||
| 183ec094c4 | |||
| c1d2b3d5c3 | |||
| be72609d1f | |||
| 48f2c98052 | |||
| 286c663495 | |||
| 941fed8e4a | |||
| 55092bede1 | |||
| 90a3b830f4 | |||
| 23b9df0694 | |||
| 48cb271a46 | |||
| 61229b38f7 | |||
| 90ca73af13 | |||
| a04b6561a0 | |||
| 219d841496 | |||
| baef49b4a2 | |||
| 3022f60561 | |||
| 680fdf6f90 | |||
| 1af6e8cb72 | |||
| 98d3913a1e | |||
| 35c38e4bd8 | |||
| e43d3a075c | |||
| 43e0ef4248 | |||
| cd7d5b59e5 | |||
| 299ef9b188 | |||
| b30c436715 | |||
| 7001fe407f | |||
| 59f7266081 |
@@ -1,33 +1,19 @@
|
||||
<!--- Provide a general summary of your changes in the Title above -->
|
||||
|
||||
## Description
|
||||
<!--- Describe your changes in detail -->
|
||||
|
||||
|
||||
## Motivation and Context
|
||||
<!--- Why is this change required? What problem does it solve? -->
|
||||
<!--- If it fixes an open issue, please link to the issue here. -->
|
||||
|
||||
## Regression
|
||||
<!-- Is this PR fixing a regression? (Yes / No) -->
|
||||
<!-- If Yes, optionally please include minio version or commit id or PR# that caused this regression, if you have these details. -->
|
||||
|
||||
## How Has This Been Tested?
|
||||
<!--- Please describe in detail how you tested your changes. -->
|
||||
<!--- Include details of your testing environment, and the tests you ran to -->
|
||||
<!--- see how your change affects other areas of the code, etc. -->
|
||||
## How to test this PR?
|
||||
|
||||
|
||||
## Types of changes
|
||||
<!--- What types of changes does your code introduce? Put an `x` in all the boxes that apply: -->
|
||||
- [ ] Bug fix (non-breaking change which fixes an issue)
|
||||
- [ ] New feature (non-breaking change which adds functionality)
|
||||
- [ ] Breaking change (fix or feature that would cause existing functionality to change)
|
||||
|
||||
## Checklist:
|
||||
<!--- Go over all the following points, and put an `x` in all the boxes that apply. -->
|
||||
<!--- If you're unsure about any of these, don't hesitate to ask. We're here to help! -->
|
||||
- [ ] My code follows the code style of this project.
|
||||
- [ ] My change requires a change to the documentation.
|
||||
- [ ] I have updated the documentation accordingly.
|
||||
- [ ] I have added unit tests to cover my changes.
|
||||
- [ ] I have added/updated functional tests in [mint](https://github.com/minio/mint). (If yes, add `mint` PR # here: )
|
||||
- [ ] All new and existing tests passed.
|
||||
- [ ] Fixes a regression (If yes, please add `commit-id` or `PR #` here)
|
||||
- [ ] Documentation needed
|
||||
- [ ] Unit tests needed
|
||||
- [ ] Functional tests needed (If yes, add [mint](https://github.com/minio/mint) PR # here: )
|
||||
|
||||
@@ -31,3 +31,10 @@ export const SHARE_OBJECT_EXPIRY_MINUTES = 0
|
||||
|
||||
export const ACCESS_KEY_MIN_LENGTH = 3
|
||||
export const SECRET_KEY_MIN_LENGTH = 8
|
||||
|
||||
export const SORT_BY_NAME = "name"
|
||||
export const SORT_BY_SIZE = "size"
|
||||
export const SORT_BY_LAST_MODIFIED = "last-modified"
|
||||
|
||||
export const SORT_ORDER_ASC = "asc"
|
||||
export const SORT_ORDER_DESC = "desc"
|
||||
|
||||
@@ -51,7 +51,7 @@ export const ObjectItem = ({
|
||||
</div>
|
||||
<div className="fesl-item fesl-item-name">
|
||||
<a
|
||||
href="#"
|
||||
href={getDataType(name, contentType) === "folder" ? name : "#"}
|
||||
onClick={e => {
|
||||
e.preventDefault()
|
||||
if (onClick) {
|
||||
|
||||
@@ -18,11 +18,19 @@ import React from "react"
|
||||
import classNames from "classnames"
|
||||
import { connect } from "react-redux"
|
||||
import * as actionsObjects from "./actions"
|
||||
import {
|
||||
SORT_BY_NAME,
|
||||
SORT_BY_SIZE,
|
||||
SORT_BY_LAST_MODIFIED,
|
||||
SORT_ORDER_DESC,
|
||||
SORT_ORDER_ASC
|
||||
} from "../constants"
|
||||
|
||||
export const ObjectsHeader = ({
|
||||
sortNameOrder,
|
||||
sortSizeOrder,
|
||||
sortLastModifiedOrder,
|
||||
sortedByName,
|
||||
sortedBySize,
|
||||
sortedByLastModified,
|
||||
sortOrder,
|
||||
sortObjects
|
||||
}) => (
|
||||
<div className="feb-container">
|
||||
@@ -31,48 +39,54 @@ export const ObjectsHeader = ({
|
||||
<div
|
||||
className="fesl-item fesl-item-name"
|
||||
id="sort-by-name"
|
||||
onClick={() => sortObjects("name")}
|
||||
onClick={() => sortObjects(SORT_BY_NAME)}
|
||||
data-sort="name"
|
||||
>
|
||||
Name
|
||||
<i
|
||||
className={classNames({
|
||||
"fesli-sort": true,
|
||||
"fesli-sort--active": sortedByName,
|
||||
fa: true,
|
||||
"fa-sort-alpha-desc": sortNameOrder,
|
||||
"fa-sort-alpha-asc": !sortNameOrder
|
||||
"fa-sort-alpha-desc": sortedByName && sortOrder === SORT_ORDER_DESC,
|
||||
"fa-sort-alpha-asc": sortedByName && sortOrder === SORT_ORDER_ASC
|
||||
})}
|
||||
/>
|
||||
</div>
|
||||
<div
|
||||
className="fesl-item fesl-item-size"
|
||||
id="sort-by-size"
|
||||
onClick={() => sortObjects("size")}
|
||||
onClick={() => sortObjects(SORT_BY_SIZE)}
|
||||
data-sort="size"
|
||||
>
|
||||
Size
|
||||
<i
|
||||
className={classNames({
|
||||
"fesli-sort": true,
|
||||
"fesli-sort--active": sortedBySize,
|
||||
fa: true,
|
||||
"fa-sort-amount-desc": sortSizeOrder,
|
||||
"fa-sort-amount-asc": !sortSizeOrder
|
||||
"fa-sort-amount-desc":
|
||||
sortedBySize && sortOrder === SORT_ORDER_DESC,
|
||||
"fa-sort-amount-asc": sortedBySize && sortOrder === SORT_ORDER_ASC
|
||||
})}
|
||||
/>
|
||||
</div>
|
||||
<div
|
||||
className="fesl-item fesl-item-modified"
|
||||
id="sort-by-last-modified"
|
||||
onClick={() => sortObjects("last-modified")}
|
||||
onClick={() => sortObjects(SORT_BY_LAST_MODIFIED)}
|
||||
data-sort="last-modified"
|
||||
>
|
||||
Last Modified
|
||||
<i
|
||||
className={classNames({
|
||||
"fesli-sort": true,
|
||||
"fesli-sort--active": sortedByLastModified,
|
||||
fa: true,
|
||||
"fa-sort-numeric-desc": sortLastModifiedOrder,
|
||||
"fa-sort-numeric-asc": !sortLastModifiedOrder
|
||||
"fa-sort-numeric-desc":
|
||||
sortedByLastModified && sortOrder === SORT_ORDER_DESC,
|
||||
"fa-sort-numeric-asc":
|
||||
sortedByLastModified && sortOrder === SORT_ORDER_ASC
|
||||
})}
|
||||
/>
|
||||
</div>
|
||||
@@ -83,10 +97,10 @@ export const ObjectsHeader = ({
|
||||
|
||||
const mapStateToProps = state => {
|
||||
return {
|
||||
sortNameOrder: state.objects.sortBy == "name" && state.objects.sortOrder,
|
||||
sortSizeOrder: state.objects.sortBy == "size" && state.objects.sortOrder,
|
||||
sortLastModifiedOrder:
|
||||
state.objects.sortBy == "last-modified" && state.objects.sortOrder
|
||||
sortedByName: state.objects.sortBy == SORT_BY_NAME,
|
||||
sortedBySize: state.objects.sortBy == SORT_BY_SIZE,
|
||||
sortedByLastModified: state.objects.sortBy == SORT_BY_LAST_MODIFIED,
|
||||
sortOrder: state.objects.sortOrder
|
||||
}
|
||||
}
|
||||
|
||||
@@ -96,4 +110,7 @@ const mapDispatchToProps = dispatch => {
|
||||
}
|
||||
}
|
||||
|
||||
export default connect(mapStateToProps, mapDispatchToProps)(ObjectsHeader)
|
||||
export default connect(
|
||||
mapStateToProps,
|
||||
mapDispatchToProps
|
||||
)(ObjectsHeader)
|
||||
|
||||
@@ -15,32 +15,52 @@
|
||||
*/
|
||||
|
||||
import React from "react"
|
||||
import classNames from "classnames"
|
||||
import { connect } from "react-redux"
|
||||
import InfiniteScroll from "react-infinite-scroller"
|
||||
import * as actionsObjects from "./actions"
|
||||
import ObjectsList from "./ObjectsList"
|
||||
|
||||
export class ObjectsListContainer extends React.Component {
|
||||
constructor(props) {
|
||||
super(props)
|
||||
this.state = {
|
||||
page: 1
|
||||
}
|
||||
this.loadNextPage = this.loadNextPage.bind(this)
|
||||
}
|
||||
componentWillReceiveProps(nextProps) {
|
||||
if (
|
||||
nextProps.currentBucket !== this.props.currentBucket ||
|
||||
nextProps.currentPrefix !== this.props.currentPrefix ||
|
||||
nextProps.sortBy !== this.props.sortBy ||
|
||||
nextProps.sortOrder !== this.props.sortOrder
|
||||
) {
|
||||
this.setState({
|
||||
page: 1
|
||||
})
|
||||
}
|
||||
}
|
||||
loadNextPage() {
|
||||
this.setState(state => {
|
||||
return { page: state.page + 1 }
|
||||
})
|
||||
}
|
||||
render() {
|
||||
const { objects, isTruncated, currentBucket, loadObjects } = this.props
|
||||
const { objects, listLoading } = this.props
|
||||
|
||||
const visibleObjects = objects.slice(0, this.state.page * 100)
|
||||
|
||||
return (
|
||||
<div className="feb-container">
|
||||
<div style={{ position: "relative" }}>
|
||||
<InfiniteScroll
|
||||
pageStart={0}
|
||||
loadMore={() => loadObjects(true)}
|
||||
hasMore={isTruncated}
|
||||
loadMore={this.loadNextPage}
|
||||
hasMore={objects.length > visibleObjects.length}
|
||||
useWindow={true}
|
||||
initialLoad={false}
|
||||
>
|
||||
<ObjectsList objects={objects} />
|
||||
<ObjectsList objects={visibleObjects} />
|
||||
</InfiniteScroll>
|
||||
<div
|
||||
className="text-center"
|
||||
style={{ display: isTruncated && currentBucket ? "block" : "none" }}
|
||||
>
|
||||
<span>Loading...</span>
|
||||
</div>
|
||||
{listLoading && <div className="loading" />}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -51,16 +71,10 @@ const mapStateToProps = state => {
|
||||
currentBucket: state.buckets.currentBucket,
|
||||
currentPrefix: state.objects.currentPrefix,
|
||||
objects: state.objects.list,
|
||||
isTruncated: state.objects.isTruncated
|
||||
sortBy: state.objects.sortBy,
|
||||
sortOrder: state.objects.sortOrder,
|
||||
listLoading: state.objects.listLoading
|
||||
}
|
||||
}
|
||||
|
||||
const mapDispatchToProps = dispatch => {
|
||||
return {
|
||||
loadObjects: append => dispatch(actionsObjects.fetchObjects(append))
|
||||
}
|
||||
}
|
||||
|
||||
export default connect(mapStateToProps, mapDispatchToProps)(
|
||||
ObjectsListContainer
|
||||
)
|
||||
export default connect(mapStateToProps)(ObjectsListContainer)
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
import React from "react"
|
||||
import { shallow } from "enzyme"
|
||||
import { ObjectsHeader } from "../ObjectsHeader"
|
||||
import { SORT_ORDER_ASC, SORT_ORDER_DESC } from "../../constants"
|
||||
|
||||
describe("ObjectsHeader", () => {
|
||||
it("should render without crashing", () => {
|
||||
@@ -24,44 +25,84 @@ describe("ObjectsHeader", () => {
|
||||
shallow(<ObjectsHeader sortObjects={sortObjects} />)
|
||||
})
|
||||
|
||||
it("should render columns with asc classes by default", () => {
|
||||
it("should render the name column with asc class when objects are sorted by name asc", () => {
|
||||
const sortObjects = jest.fn()
|
||||
const wrapper = shallow(<ObjectsHeader sortObjects={sortObjects} />)
|
||||
const wrapper = shallow(
|
||||
<ObjectsHeader
|
||||
sortObjects={sortObjects}
|
||||
sortedByName={true}
|
||||
sortOrder={SORT_ORDER_ASC}
|
||||
/>
|
||||
)
|
||||
expect(
|
||||
wrapper.find("#sort-by-name i").hasClass("fa-sort-alpha-asc")
|
||||
).toBeTruthy()
|
||||
expect(
|
||||
wrapper.find("#sort-by-size i").hasClass("fa-sort-amount-asc")
|
||||
).toBeTruthy()
|
||||
expect(
|
||||
wrapper.find("#sort-by-last-modified i").hasClass("fa-sort-numeric-asc")
|
||||
).toBeTruthy()
|
||||
})
|
||||
|
||||
it("should render name column with desc class when objects are sorted by name", () => {
|
||||
it("should render the name column with desc class when objects are sorted by name desc", () => {
|
||||
const sortObjects = jest.fn()
|
||||
const wrapper = shallow(
|
||||
<ObjectsHeader sortObjects={sortObjects} sortNameOrder={true} />
|
||||
<ObjectsHeader
|
||||
sortObjects={sortObjects}
|
||||
sortedByName={true}
|
||||
sortOrder={SORT_ORDER_DESC}
|
||||
/>
|
||||
)
|
||||
expect(
|
||||
wrapper.find("#sort-by-name i").hasClass("fa-sort-alpha-desc")
|
||||
).toBeTruthy()
|
||||
})
|
||||
|
||||
it("should render size column with desc class when objects are sorted by size", () => {
|
||||
it("should render the size column with asc class when objects are sorted by size asc", () => {
|
||||
const sortObjects = jest.fn()
|
||||
const wrapper = shallow(
|
||||
<ObjectsHeader sortObjects={sortObjects} sortSizeOrder={true} />
|
||||
<ObjectsHeader
|
||||
sortObjects={sortObjects}
|
||||
sortedBySize={true}
|
||||
sortOrder={SORT_ORDER_ASC}
|
||||
/>
|
||||
)
|
||||
expect(
|
||||
wrapper.find("#sort-by-size i").hasClass("fa-sort-amount-asc")
|
||||
).toBeTruthy()
|
||||
})
|
||||
|
||||
it("should render the size column with desc class when objects are sorted by size desc", () => {
|
||||
const sortObjects = jest.fn()
|
||||
const wrapper = shallow(
|
||||
<ObjectsHeader
|
||||
sortObjects={sortObjects}
|
||||
sortedBySize={true}
|
||||
sortOrder={SORT_ORDER_DESC}
|
||||
/>
|
||||
)
|
||||
expect(
|
||||
wrapper.find("#sort-by-size i").hasClass("fa-sort-amount-desc")
|
||||
).toBeTruthy()
|
||||
})
|
||||
|
||||
it("should render last modified column with desc class when objects are sorted by last modified", () => {
|
||||
it("should render the date column with asc class when objects are sorted by date asc", () => {
|
||||
const sortObjects = jest.fn()
|
||||
const wrapper = shallow(
|
||||
<ObjectsHeader sortObjects={sortObjects} sortLastModifiedOrder={true} />
|
||||
<ObjectsHeader
|
||||
sortObjects={sortObjects}
|
||||
sortedByLastModified={true}
|
||||
sortOrder={SORT_ORDER_ASC}
|
||||
/>
|
||||
)
|
||||
expect(
|
||||
wrapper.find("#sort-by-last-modified i").hasClass("fa-sort-numeric-asc")
|
||||
).toBeTruthy()
|
||||
})
|
||||
|
||||
it("should render the date column with desc class when objects are sorted by date desc", () => {
|
||||
const sortObjects = jest.fn()
|
||||
const wrapper = shallow(
|
||||
<ObjectsHeader
|
||||
sortObjects={sortObjects}
|
||||
sortedByLastModified={true}
|
||||
sortOrder={SORT_ORDER_DESC}
|
||||
/>
|
||||
)
|
||||
expect(
|
||||
wrapper.find("#sort-by-last-modified i").hasClass("fa-sort-numeric-desc")
|
||||
|
||||
@@ -20,14 +20,13 @@ import { ObjectsListContainer } from "../ObjectsListContainer"
|
||||
|
||||
describe("ObjectsList", () => {
|
||||
it("should render without crashing", () => {
|
||||
shallow(<ObjectsListContainer loadObjects={jest.fn()} />)
|
||||
shallow(<ObjectsListContainer objects={[]} />)
|
||||
})
|
||||
|
||||
it("should render ObjectsList with objects", () => {
|
||||
const wrapper = shallow(
|
||||
<ObjectsListContainer
|
||||
objects={[{ name: "test1.jpg" }, { name: "test2.jpg" }]}
|
||||
loadObjects={jest.fn()}
|
||||
/>
|
||||
)
|
||||
expect(wrapper.find("ObjectsList").length).toBe(1)
|
||||
@@ -37,10 +36,14 @@ describe("ObjectsList", () => {
|
||||
])
|
||||
})
|
||||
|
||||
it("should show the loading indicator at the bottom if there are more elements to display", () => {
|
||||
it("should show the loading indicator when the objects are being loaded", () => {
|
||||
const wrapper = shallow(
|
||||
<ObjectsListContainer currentBucket="test1" isTruncated={true} />
|
||||
<ObjectsListContainer
|
||||
currentBucket="test1"
|
||||
objects={[]}
|
||||
listLoading={true}
|
||||
/>
|
||||
)
|
||||
expect(wrapper.find(".text-center").prop("style")).toHaveProperty("display", "block")
|
||||
expect(wrapper.find(".loading").exists()).toBeTruthy()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -18,7 +18,13 @@ import configureStore from "redux-mock-store"
|
||||
import thunk from "redux-thunk"
|
||||
import * as actionsObjects from "../actions"
|
||||
import * as alertActions from "../../alert/actions"
|
||||
import { minioBrowserPrefix } from "../../constants"
|
||||
import {
|
||||
minioBrowserPrefix,
|
||||
SORT_BY_NAME,
|
||||
SORT_ORDER_ASC,
|
||||
SORT_BY_LAST_MODIFIED,
|
||||
SORT_ORDER_DESC
|
||||
} from "../../constants"
|
||||
import history from "../../history"
|
||||
|
||||
jest.mock("../../web", () => ({
|
||||
@@ -37,8 +43,6 @@ jest.mock("../../web", () => ({
|
||||
} else {
|
||||
return Promise.resolve({
|
||||
objects: [{ name: "test1" }, { name: "test2" }],
|
||||
istruncated: false,
|
||||
nextmarker: "test2",
|
||||
writable: false
|
||||
})
|
||||
}
|
||||
@@ -77,17 +81,11 @@ describe("Objects actions", () => {
|
||||
const expectedActions = [
|
||||
{
|
||||
type: "objects/SET_LIST",
|
||||
objects: [{ name: "test1" }, { name: "test2" }],
|
||||
isTruncated: false,
|
||||
marker: "test2"
|
||||
objects: [{ name: "test1" }, { name: "test2" }]
|
||||
}
|
||||
]
|
||||
store.dispatch(
|
||||
actionsObjects.setList(
|
||||
[{ name: "test1" }, { name: "test2" }],
|
||||
"test2",
|
||||
false
|
||||
)
|
||||
actionsObjects.setList([{ name: "test1" }, { name: "test2" }])
|
||||
)
|
||||
const actions = store.getActions()
|
||||
expect(actions).toEqual(expectedActions)
|
||||
@@ -98,10 +96,10 @@ describe("Objects actions", () => {
|
||||
const expectedActions = [
|
||||
{
|
||||
type: "objects/SET_SORT_BY",
|
||||
sortBy: "name"
|
||||
sortBy: SORT_BY_NAME
|
||||
}
|
||||
]
|
||||
store.dispatch(actionsObjects.setSortBy("name"))
|
||||
store.dispatch(actionsObjects.setSortBy(SORT_BY_NAME))
|
||||
const actions = store.getActions()
|
||||
expect(actions).toEqual(expectedActions)
|
||||
})
|
||||
@@ -111,10 +109,10 @@ describe("Objects actions", () => {
|
||||
const expectedActions = [
|
||||
{
|
||||
type: "objects/SET_SORT_ORDER",
|
||||
sortOrder: true
|
||||
sortOrder: SORT_ORDER_ASC
|
||||
}
|
||||
]
|
||||
store.dispatch(actionsObjects.setSortOrder(true))
|
||||
store.dispatch(actionsObjects.setSortOrder(SORT_ORDER_ASC))
|
||||
const actions = store.getActions()
|
||||
expect(actions).toEqual(expectedActions)
|
||||
})
|
||||
@@ -126,23 +124,26 @@ describe("Objects actions", () => {
|
||||
})
|
||||
const expectedActions = [
|
||||
{
|
||||
type: "objects/SET_LIST",
|
||||
objects: [{ name: "test1" }, { name: "test2" }],
|
||||
marker: "test2",
|
||||
isTruncated: false
|
||||
type: "objects/RESET_LIST"
|
||||
},
|
||||
{ listLoading: true, type: "objects/SET_LIST_LOADING" },
|
||||
{
|
||||
type: "objects/SET_SORT_BY",
|
||||
sortBy: ""
|
||||
sortBy: SORT_BY_LAST_MODIFIED
|
||||
},
|
||||
{
|
||||
type: "objects/SET_SORT_ORDER",
|
||||
sortOrder: false
|
||||
sortOrder: SORT_ORDER_DESC
|
||||
},
|
||||
{
|
||||
type: "objects/SET_LIST",
|
||||
objects: [{ name: "test2" }, { name: "test1" }]
|
||||
},
|
||||
{
|
||||
type: "objects/SET_PREFIX_WRITABLE",
|
||||
prefixWritable: false
|
||||
}
|
||||
},
|
||||
{ listLoading: false, type: "objects/SET_LIST_LOADING" }
|
||||
]
|
||||
return store.dispatch(actionsObjects.fetchObjects()).then(() => {
|
||||
const actions = store.getActions()
|
||||
@@ -150,35 +151,16 @@ describe("Objects actions", () => {
|
||||
})
|
||||
})
|
||||
|
||||
it("creates objects/APPEND_LIST after fetching more objects", () => {
|
||||
const store = mockStore({
|
||||
buckets: { currentBucket: "bk1" },
|
||||
objects: { currentPrefix: "" }
|
||||
})
|
||||
const expectedActions = [
|
||||
{
|
||||
type: "objects/APPEND_LIST",
|
||||
objects: [{ name: "test1" }, { name: "test2" }],
|
||||
marker: "test2",
|
||||
isTruncated: false
|
||||
},
|
||||
{
|
||||
type: "objects/SET_PREFIX_WRITABLE",
|
||||
prefixWritable: false
|
||||
}
|
||||
]
|
||||
return store.dispatch(actionsObjects.fetchObjects(true)).then(() => {
|
||||
const actions = store.getActions()
|
||||
expect(actions).toEqual(expectedActions)
|
||||
})
|
||||
})
|
||||
|
||||
it("creates objects/RESET_LIST after failing to fetch the objects from bucket with ListObjects denied for LoggedIn users", () => {
|
||||
const store = mockStore({
|
||||
buckets: { currentBucket: "test-deny" },
|
||||
objects: { currentPrefix: "" }
|
||||
})
|
||||
const expectedActions = [
|
||||
{
|
||||
type: "objects/RESET_LIST"
|
||||
},
|
||||
{ listLoading: true, type: "objects/SET_LIST_LOADING" },
|
||||
{
|
||||
type: "alert/SET",
|
||||
alert: {
|
||||
@@ -189,8 +171,9 @@ describe("Objects actions", () => {
|
||||
}
|
||||
},
|
||||
{
|
||||
type: "object/RESET_LIST"
|
||||
}
|
||||
type: "objects/RESET_LIST"
|
||||
},
|
||||
{ listLoading: false, type: "objects/SET_LIST_LOADING" }
|
||||
]
|
||||
return store.dispatch(actionsObjects.fetchObjects()).then(() => {
|
||||
const actions = store.getActions()
|
||||
@@ -213,28 +196,24 @@ describe("Objects actions", () => {
|
||||
objects: {
|
||||
list: [],
|
||||
sortBy: "",
|
||||
sortOrder: false,
|
||||
isTruncated: false,
|
||||
marker: ""
|
||||
sortOrder: SORT_ORDER_ASC
|
||||
}
|
||||
})
|
||||
const expectedActions = [
|
||||
{
|
||||
type: "objects/SET_SORT_BY",
|
||||
sortBy: "name"
|
||||
sortBy: SORT_BY_NAME
|
||||
},
|
||||
{
|
||||
type: "objects/SET_SORT_ORDER",
|
||||
sortOrder: true
|
||||
sortOrder: SORT_ORDER_ASC
|
||||
},
|
||||
{
|
||||
type: "objects/SET_LIST",
|
||||
objects: [],
|
||||
isTruncated: false,
|
||||
marker: ""
|
||||
objects: []
|
||||
}
|
||||
]
|
||||
store.dispatch(actionsObjects.sortObjects("name"))
|
||||
store.dispatch(actionsObjects.sortObjects(SORT_BY_NAME))
|
||||
const actions = store.getActions()
|
||||
expect(actions).toEqual(expectedActions)
|
||||
})
|
||||
@@ -246,6 +225,10 @@ describe("Objects actions", () => {
|
||||
})
|
||||
const expectedActions = [
|
||||
{ type: "objects/SET_CURRENT_PREFIX", prefix: "abc/" },
|
||||
{
|
||||
type: "objects/RESET_LIST"
|
||||
},
|
||||
{ listLoading: true, type: "objects/SET_LIST_LOADING" },
|
||||
{ type: "objects/CHECKED_LIST_RESET" }
|
||||
]
|
||||
store.dispatch(actionsObjects.selectPrefix("abc/"))
|
||||
|
||||
@@ -16,17 +16,17 @@
|
||||
|
||||
import reducer from "../reducer"
|
||||
import * as actions from "../actions"
|
||||
import { SORT_ORDER_ASC, SORT_BY_NAME } from "../../constants"
|
||||
|
||||
describe("objects reducer", () => {
|
||||
it("should return the initial state", () => {
|
||||
const initialState = reducer(undefined, {})
|
||||
expect(initialState).toEqual({
|
||||
list: [],
|
||||
listLoading: false,
|
||||
sortBy: "",
|
||||
sortOrder: false,
|
||||
sortOrder: SORT_ORDER_ASC,
|
||||
currentPrefix: "",
|
||||
marker: "",
|
||||
isTruncated: false,
|
||||
prefixWritable: false,
|
||||
shareObject: {
|
||||
show: false,
|
||||
@@ -40,37 +40,9 @@ describe("objects reducer", () => {
|
||||
it("should handle SET_LIST", () => {
|
||||
const newState = reducer(undefined, {
|
||||
type: actions.SET_LIST,
|
||||
objects: [{ name: "obj1" }, { name: "obj2" }],
|
||||
marker: "obj2",
|
||||
isTruncated: false
|
||||
objects: [{ name: "obj1" }, { name: "obj2" }]
|
||||
})
|
||||
expect(newState.list).toEqual([{ name: "obj1" }, { name: "obj2" }])
|
||||
expect(newState.marker).toBe("obj2")
|
||||
expect(newState.isTruncated).toBeFalsy()
|
||||
})
|
||||
|
||||
it("should handle APPEND_LIST", () => {
|
||||
const newState = reducer(
|
||||
{
|
||||
list: [{ name: "obj1" }, { name: "obj2" }],
|
||||
marker: "obj2",
|
||||
isTruncated: true
|
||||
},
|
||||
{
|
||||
type: actions.APPEND_LIST,
|
||||
objects: [{ name: "obj3" }, { name: "obj4" }],
|
||||
marker: "obj4",
|
||||
isTruncated: false
|
||||
}
|
||||
)
|
||||
expect(newState.list).toEqual([
|
||||
{ name: "obj1" },
|
||||
{ name: "obj2" },
|
||||
{ name: "obj3" },
|
||||
{ name: "obj4" }
|
||||
])
|
||||
expect(newState.marker).toBe("obj4")
|
||||
expect(newState.isTruncated).toBeFalsy()
|
||||
})
|
||||
|
||||
it("should handle REMOVE", () => {
|
||||
@@ -98,30 +70,28 @@ describe("objects reducer", () => {
|
||||
it("should handle SET_SORT_BY", () => {
|
||||
const newState = reducer(undefined, {
|
||||
type: actions.SET_SORT_BY,
|
||||
sortBy: "name"
|
||||
sortBy: SORT_BY_NAME
|
||||
})
|
||||
expect(newState.sortBy).toEqual("name")
|
||||
expect(newState.sortBy).toEqual(SORT_BY_NAME)
|
||||
})
|
||||
|
||||
it("should handle SET_SORT_ORDER", () => {
|
||||
const newState = reducer(undefined, {
|
||||
type: actions.SET_SORT_ORDER,
|
||||
sortOrder: true
|
||||
sortOrder: SORT_ORDER_ASC
|
||||
})
|
||||
expect(newState.sortOrder).toEqual(true)
|
||||
expect(newState.sortOrder).toEqual(SORT_ORDER_ASC)
|
||||
})
|
||||
|
||||
it("should handle SET_CURRENT_PREFIX", () => {
|
||||
const newState = reducer(
|
||||
{ currentPrefix: "test1/", marker: "abc", isTruncated: true },
|
||||
{ currentPrefix: "test1/" },
|
||||
{
|
||||
type: actions.SET_CURRENT_PREFIX,
|
||||
prefix: "test2/"
|
||||
}
|
||||
)
|
||||
expect(newState.currentPrefix).toEqual("test2/")
|
||||
expect(newState.marker).toEqual("")
|
||||
expect(newState.isTruncated).toBeFalsy()
|
||||
})
|
||||
|
||||
it("should handle SET_PREFIX_WRITABLE", () => {
|
||||
|
||||
@@ -16,15 +16,26 @@
|
||||
|
||||
import web from "../web"
|
||||
import history from "../history"
|
||||
import { sortObjectsByName, sortObjectsBySize, sortObjectsByDate } from "../utils"
|
||||
import {
|
||||
sortObjectsByName,
|
||||
sortObjectsBySize,
|
||||
sortObjectsByDate
|
||||
} from "../utils"
|
||||
import { getCurrentBucket } from "../buckets/selectors"
|
||||
import { getCurrentPrefix, getCheckedList } from "./selectors"
|
||||
import * as alertActions from "../alert/actions"
|
||||
import * as bucketActions from "../buckets/actions"
|
||||
import { minioBrowserPrefix } from "../constants"
|
||||
import {
|
||||
minioBrowserPrefix,
|
||||
SORT_BY_NAME,
|
||||
SORT_BY_SIZE,
|
||||
SORT_BY_LAST_MODIFIED,
|
||||
SORT_ORDER_ASC,
|
||||
SORT_ORDER_DESC
|
||||
} from "../constants"
|
||||
|
||||
export const SET_LIST = "objects/SET_LIST"
|
||||
export const RESET_LIST = "object/RESET_LIST"
|
||||
export const RESET_LIST = "objects/RESET_LIST"
|
||||
export const APPEND_LIST = "objects/APPEND_LIST"
|
||||
export const REMOVE = "objects/REMOVE"
|
||||
export const SET_SORT_BY = "objects/SET_SORT_BY"
|
||||
@@ -35,53 +46,63 @@ export const SET_SHARE_OBJECT = "objects/SET_SHARE_OBJECT"
|
||||
export const CHECKED_LIST_ADD = "objects/CHECKED_LIST_ADD"
|
||||
export const CHECKED_LIST_REMOVE = "objects/CHECKED_LIST_REMOVE"
|
||||
export const CHECKED_LIST_RESET = "objects/CHECKED_LIST_RESET"
|
||||
export const SET_LIST_LOADING = "objects/SET_LIST_LOADING"
|
||||
|
||||
export const setList = (objects, marker, isTruncated) => ({
|
||||
export const setList = objects => ({
|
||||
type: SET_LIST,
|
||||
objects,
|
||||
marker,
|
||||
isTruncated
|
||||
objects
|
||||
})
|
||||
|
||||
export const resetList = () => ({
|
||||
type: RESET_LIST
|
||||
})
|
||||
|
||||
export const appendList = (objects, marker, isTruncated) => ({
|
||||
type: APPEND_LIST,
|
||||
objects,
|
||||
marker,
|
||||
isTruncated
|
||||
export const setListLoading = listLoading => ({
|
||||
type: SET_LIST_LOADING,
|
||||
listLoading
|
||||
})
|
||||
|
||||
export const fetchObjects = append => {
|
||||
export const fetchObjects = () => {
|
||||
return function(dispatch, getState) {
|
||||
const {buckets: {currentBucket}, objects: {currentPrefix, marker}} = getState()
|
||||
dispatch(resetList())
|
||||
const {
|
||||
buckets: { currentBucket },
|
||||
objects: { currentPrefix }
|
||||
} = getState()
|
||||
if (currentBucket) {
|
||||
dispatch(setListLoading(true))
|
||||
return web
|
||||
.ListObjects({
|
||||
bucketName: currentBucket,
|
||||
prefix: currentPrefix,
|
||||
marker: append ? marker : ""
|
||||
prefix: currentPrefix
|
||||
})
|
||||
.then(res => {
|
||||
let objects = []
|
||||
if (res.objects) {
|
||||
objects = res.objects.map(object => {
|
||||
return {
|
||||
...object,
|
||||
name: object.name.replace(currentPrefix, "")
|
||||
}
|
||||
})
|
||||
// we need to check if the bucket name and prefix are the same as
|
||||
// when the request was made before updating the displayed objects
|
||||
if (
|
||||
currentBucket === getCurrentBucket(getState()) &&
|
||||
currentPrefix === getCurrentPrefix(getState())
|
||||
) {
|
||||
let objects = []
|
||||
if (res.objects) {
|
||||
objects = res.objects.map(object => {
|
||||
return {
|
||||
...object,
|
||||
name: object.name.replace(currentPrefix, "")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
const sortBy = SORT_BY_LAST_MODIFIED
|
||||
const sortOrder = SORT_ORDER_DESC
|
||||
dispatch(setSortBy(sortBy))
|
||||
dispatch(setSortOrder(sortOrder))
|
||||
const sortedList = sortObjectsList(objects, sortBy, sortOrder)
|
||||
dispatch(setList(sortedList))
|
||||
|
||||
dispatch(setPrefixWritable(res.writable))
|
||||
dispatch(setListLoading(false))
|
||||
}
|
||||
if (append) {
|
||||
dispatch(appendList(objects, res.nextmarker, res.istruncated))
|
||||
} else {
|
||||
dispatch(setList(objects, res.nextmarker, res.istruncated))
|
||||
dispatch(setSortBy(""))
|
||||
dispatch(setSortOrder(false))
|
||||
}
|
||||
dispatch(setPrefixWritable(res.writable))
|
||||
})
|
||||
.catch(err => {
|
||||
if (web.LoggedIn()) {
|
||||
@@ -96,6 +117,7 @@ export const fetchObjects = append => {
|
||||
} else {
|
||||
history.push("/login")
|
||||
}
|
||||
dispatch(setListLoading(false))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -103,26 +125,27 @@ export const fetchObjects = append => {
|
||||
|
||||
export const sortObjects = sortBy => {
|
||||
return function(dispatch, getState) {
|
||||
const {objects} = getState()
|
||||
const sortOrder = objects.sortBy == sortBy ? !objects.sortOrder : true
|
||||
const { objects } = getState()
|
||||
let sortOrder = SORT_ORDER_ASC
|
||||
// Reverse sort order if the list is already sorted on same field
|
||||
if (objects.sortBy === sortBy && objects.sortOrder === SORT_ORDER_ASC) {
|
||||
sortOrder = SORT_ORDER_DESC
|
||||
}
|
||||
dispatch(setSortBy(sortBy))
|
||||
dispatch(setSortOrder(sortOrder))
|
||||
let list
|
||||
switch (sortBy) {
|
||||
case "name":
|
||||
list = sortObjectsByName(objects.list, sortOrder)
|
||||
break
|
||||
case "size":
|
||||
list = sortObjectsBySize(objects.list, sortOrder)
|
||||
break
|
||||
case "last-modified":
|
||||
list = sortObjectsByDate(objects.list, sortOrder)
|
||||
break
|
||||
default:
|
||||
list = objects.list
|
||||
break
|
||||
}
|
||||
dispatch(setList(list, objects.marker, objects.isTruncated))
|
||||
const sortedList = sortObjectsList(objects.list, sortBy, sortOrder)
|
||||
dispatch(setList(sortedList))
|
||||
}
|
||||
}
|
||||
|
||||
const sortObjectsList = (list, sortBy, sortOrder) => {
|
||||
switch (sortBy) {
|
||||
case SORT_BY_NAME:
|
||||
return sortObjectsByName(list, sortOrder)
|
||||
case SORT_BY_SIZE:
|
||||
return sortObjectsBySize(list, sortOrder)
|
||||
case SORT_BY_LAST_MODIFIED:
|
||||
return sortObjectsByDate(list, sortOrder)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -229,7 +252,16 @@ export const shareObject = (object, days, hours, minutes) => {
|
||||
)
|
||||
})
|
||||
} else {
|
||||
dispatch(showShareObject(object, `${location.host}` + '/' + `${currentBucket}` + '/' + encodeURI(objectName)))
|
||||
dispatch(
|
||||
showShareObject(
|
||||
object,
|
||||
`${location.host}` +
|
||||
"/" +
|
||||
`${currentBucket}` +
|
||||
"/" +
|
||||
encodeURI(objectName)
|
||||
)
|
||||
)
|
||||
dispatch(
|
||||
alertActions.set({
|
||||
type: "success",
|
||||
@@ -322,13 +354,14 @@ export const downloadCheckedObjects = () => {
|
||||
}${minioBrowserPrefix}/zip?token=${res.token}`
|
||||
downloadZip(requestUrl, req, dispatch)
|
||||
})
|
||||
.catch(err => dispatch(
|
||||
alertActions.set({
|
||||
type: "danger",
|
||||
message: err.message
|
||||
})
|
||||
.catch(err =>
|
||||
dispatch(
|
||||
alertActions.set({
|
||||
type: "danger",
|
||||
message: err.message
|
||||
})
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -351,7 +384,8 @@ const downloadZip = (url, req, dispatch) => {
|
||||
var separator = req.prefix.length > 1 ? "-" : ""
|
||||
|
||||
anchor.href = blobUrl
|
||||
anchor.download = req.bucketName + separator + req.prefix.slice(0, -1) + ".zip"
|
||||
anchor.download =
|
||||
req.bucketName + separator + req.prefix.slice(0, -1) + ".zip"
|
||||
|
||||
anchor.click()
|
||||
window.URL.revokeObjectURL(blobUrl)
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
*/
|
||||
|
||||
import * as actionsObjects from "./actions"
|
||||
import { SORT_ORDER_ASC } from "../constants"
|
||||
|
||||
const removeObject = (list, objectToRemove, lookup) => {
|
||||
const idx = list.findIndex(object => lookup(object) === objectToRemove)
|
||||
@@ -27,11 +28,10 @@ const removeObject = (list, objectToRemove, lookup) => {
|
||||
export default (
|
||||
state = {
|
||||
list: [],
|
||||
listLoading: false,
|
||||
sortBy: "",
|
||||
sortOrder: false,
|
||||
sortOrder: SORT_ORDER_ASC,
|
||||
currentPrefix: "",
|
||||
marker: "",
|
||||
isTruncated: false,
|
||||
prefixWritable: false,
|
||||
shareObject: {
|
||||
show: false,
|
||||
@@ -46,23 +46,17 @@ export default (
|
||||
case actionsObjects.SET_LIST:
|
||||
return {
|
||||
...state,
|
||||
list: action.objects,
|
||||
marker: action.marker,
|
||||
isTruncated: action.isTruncated
|
||||
list: action.objects
|
||||
}
|
||||
case actionsObjects.RESET_LIST:
|
||||
return {
|
||||
...state,
|
||||
list: [],
|
||||
marker: "",
|
||||
isTruncated: false
|
||||
list: []
|
||||
}
|
||||
case actionsObjects.APPEND_LIST:
|
||||
case actionsObjects.SET_LIST_LOADING:
|
||||
return {
|
||||
...state,
|
||||
list: [...state.list, ...action.objects],
|
||||
marker: action.marker,
|
||||
isTruncated: action.isTruncated
|
||||
listLoading: action.listLoading
|
||||
}
|
||||
case actionsObjects.REMOVE:
|
||||
return {
|
||||
@@ -82,9 +76,7 @@ export default (
|
||||
case actionsObjects.SET_CURRENT_PREFIX:
|
||||
return {
|
||||
...state,
|
||||
currentPrefix: action.prefix,
|
||||
marker: "",
|
||||
isTruncated: false
|
||||
currentPrefix: action.prefix
|
||||
}
|
||||
case actionsObjects.SET_PREFIX_WRITABLE:
|
||||
return {
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { minioBrowserPrefix } from "./constants.js"
|
||||
import { minioBrowserPrefix, SORT_ORDER_DESC } from "./constants.js"
|
||||
|
||||
export const sortObjectsByName = (objects, order) => {
|
||||
let folders = objects.filter(object => object.name.endsWith("/"))
|
||||
@@ -29,7 +29,7 @@ export const sortObjectsByName = (objects, order) => {
|
||||
if (a.name.toLowerCase() > b.name.toLowerCase()) return 1
|
||||
return 0
|
||||
})
|
||||
if (order) {
|
||||
if (order === SORT_ORDER_DESC) {
|
||||
folders = folders.reverse()
|
||||
files = files.reverse()
|
||||
}
|
||||
@@ -40,7 +40,7 @@ export const sortObjectsBySize = (objects, order) => {
|
||||
let folders = objects.filter(object => object.name.endsWith("/"))
|
||||
let files = objects.filter(object => !object.name.endsWith("/"))
|
||||
files = files.sort((a, b) => a.size - b.size)
|
||||
if (order) files = files.reverse()
|
||||
if (order === SORT_ORDER_DESC) files = files.reverse()
|
||||
return [...folders, ...files]
|
||||
}
|
||||
|
||||
@@ -51,7 +51,7 @@ export const sortObjectsByDate = (objects, order) => {
|
||||
(a, b) =>
|
||||
new Date(a.lastModified).getTime() - new Date(b.lastModified).getTime()
|
||||
)
|
||||
if (order) files = files.reverse()
|
||||
if (order === SORT_ORDER_DESC) files = files.reverse()
|
||||
return [...folders, ...files]
|
||||
}
|
||||
|
||||
|
||||
@@ -43,6 +43,9 @@ header.fesl-row {
|
||||
color: @dark-gray;
|
||||
font-size: 14px;
|
||||
}
|
||||
& > .fesli-sort--active {
|
||||
.opacity(0.5);
|
||||
}
|
||||
|
||||
&:hover:not(.fesl-item-actions) {
|
||||
background: lighten(@text-muted-color, 22%);
|
||||
|
||||
@@ -113,4 +113,41 @@
|
||||
margin: 0;
|
||||
vertical-align: top;
|
||||
}
|
||||
}
|
||||
|
||||
.loading {
|
||||
position: absolute;
|
||||
margin: auto;
|
||||
left: 0;
|
||||
right: 0;
|
||||
top: 0;
|
||||
bottom: 0;
|
||||
border-top: 1px solid @loading-track-bg;
|
||||
border-right: 1px solid @loading-track-bg;
|
||||
border-bottom: 1px solid @loading-track-bg;
|
||||
border-left: 1px solid @loading-point-bg;
|
||||
transform: translateZ(0);
|
||||
animation: loading 1.1s infinite linear;
|
||||
border-radius: 50%;
|
||||
width: 35px;
|
||||
height: 35px;
|
||||
margin-top: 30px;
|
||||
}
|
||||
|
||||
@-webkit-keyframes loading {
|
||||
0% {
|
||||
transform: rotate(0deg);
|
||||
}
|
||||
100% {
|
||||
transform: rotate(360deg);
|
||||
}
|
||||
}
|
||||
|
||||
@keyframes loading {
|
||||
0% {
|
||||
transform: rotate(0deg);
|
||||
}
|
||||
100% {
|
||||
transform: rotate(360deg);
|
||||
}
|
||||
}
|
||||
@@ -100,4 +100,10 @@
|
||||
List
|
||||
--------------------------*/
|
||||
@list-row-selected-bg: #fbf2bf;
|
||||
@list-row-even-bg: #fafafa;
|
||||
@list-row-even-bg: #fafafa;
|
||||
|
||||
/*--------------------------
|
||||
Loading
|
||||
---------------------------*/
|
||||
@loading-track-bg: #eeeeee;
|
||||
@loading-point-bg: #00303f;
|
||||
+35
-52
File diff suppressed because one or more lines are too long
+1
-5
@@ -8041,15 +8041,11 @@ string_decoder@~1.1.1:
|
||||
dependencies:
|
||||
safe-buffer "~5.1.0"
|
||||
|
||||
stringstream@~0.0.4:
|
||||
stringstream@~0.0.4, stringstream@~0.0.5:
|
||||
version "0.0.6"
|
||||
resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.6.tgz#7880225b0d4ad10e30927d167a1d6f2fd3b33a72"
|
||||
integrity sha512-87GEBAkegbBcweToUrdzf3eLhWNg06FJTebl4BVJz/JgWy8CvEr9dRtX5qWphiynMSQlxxi+QqN0z5T32SLlhA==
|
||||
|
||||
stringstream@~0.0.5:
|
||||
version "0.0.5"
|
||||
resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.5.tgz#4e484cd4de5a0bbbee18e46307710a8a81621878"
|
||||
|
||||
strip-ansi@^3.0.0, strip-ansi@^3.0.1:
|
||||
version "3.0.1"
|
||||
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf"
|
||||
|
||||
+95
-23
@@ -34,6 +34,7 @@ import (
|
||||
"github.com/tidwall/gjson"
|
||||
"github.com/tidwall/sjson"
|
||||
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/cpu"
|
||||
"github.com/minio/minio/pkg/disk"
|
||||
@@ -43,6 +44,7 @@ import (
|
||||
"github.com/minio/minio/pkg/mem"
|
||||
xnet "github.com/minio/minio/pkg/net"
|
||||
"github.com/minio/minio/pkg/quick"
|
||||
trace "github.com/minio/minio/pkg/trace"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -239,8 +241,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
|
||||
hostName, err := getHostName(r)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -250,7 +251,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
// Once we have received all the ServerInfo from peers
|
||||
// add the local peer server info as well.
|
||||
serverInfo = append(serverInfo, ServerInfo{
|
||||
Addr: thisAddr.String(),
|
||||
Addr: hostName,
|
||||
Data: &ServerInfoData{
|
||||
StorageInfo: objectAPI.StorageInfo(ctx),
|
||||
ConnStats: globalConnStats.toServerConnStats(),
|
||||
@@ -328,7 +329,7 @@ func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request
|
||||
return
|
||||
}
|
||||
// Get drive performance details from local server's drive(s)
|
||||
dp := localEndpointsDrivePerf(globalEndpoints)
|
||||
dp := localEndpointsDrivePerf(globalEndpoints, r)
|
||||
|
||||
// Notify all other MinIO peers to report drive performance numbers
|
||||
dps := globalNotificationSys.DrivePerfInfo()
|
||||
@@ -346,7 +347,7 @@ func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request
|
||||
writeSuccessResponseJSON(w, jsonBytes)
|
||||
case "cpu":
|
||||
// Get CPU load details from local server's cpu(s)
|
||||
cpu := localEndpointsCPULoad(globalEndpoints)
|
||||
cpu := localEndpointsCPULoad(globalEndpoints, r)
|
||||
// Notify all other MinIO peers to report cpu load numbers
|
||||
cpus := globalNotificationSys.CPULoadInfo()
|
||||
cpus = append(cpus, cpu)
|
||||
@@ -363,7 +364,7 @@ func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request
|
||||
writeSuccessResponseJSON(w, jsonBytes)
|
||||
case "mem":
|
||||
// Get mem usage details from local server(s)
|
||||
m := localEndpointsMemUsage(globalEndpoints)
|
||||
m := localEndpointsMemUsage(globalEndpoints, r)
|
||||
// Notify all other MinIO peers to report mem usage numbers
|
||||
mems := globalNotificationSys.MemUsageInfo()
|
||||
mems = append(mems, m)
|
||||
@@ -441,8 +442,7 @@ func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
|
||||
hostName, err := getHostName(r)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -453,7 +453,7 @@ func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request
|
||||
// add the local peer locks list as well.
|
||||
localLocks := globalLockServer.ll.DupLockMap()
|
||||
peerLocks = append(peerLocks, &PeerLocks{
|
||||
Addr: thisAddr.String(),
|
||||
Addr: hostName,
|
||||
Locks: localLocks,
|
||||
})
|
||||
|
||||
@@ -681,7 +681,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
// Start writing response to client
|
||||
started = true
|
||||
setCommonHeaders(w)
|
||||
w.Header().Set("Content-Type", string(mimeJSON))
|
||||
w.Header().Set(xhttp.ContentType, string(mimeJSON))
|
||||
// Set 200 OK status
|
||||
w.WriteHeader(200)
|
||||
}
|
||||
@@ -701,20 +701,20 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
var errorRespJSON []byte
|
||||
if hr.errBody == "" {
|
||||
errorRespJSON = encodeResponseJSON(getAPIErrorResponse(ctx, hr.apiErr,
|
||||
r.URL.Path, w.Header().Get(responseRequestIDKey),
|
||||
w.Header().Get(responseDeploymentIDKey)))
|
||||
r.URL.Path, w.Header().Get(xhttp.AmzRequestID),
|
||||
globalDeploymentID))
|
||||
} else {
|
||||
errorRespJSON = encodeResponseJSON(APIErrorResponse{
|
||||
Code: hr.apiErr.Code,
|
||||
Message: hr.errBody,
|
||||
Resource: r.URL.Path,
|
||||
RequestID: w.Header().Get(responseRequestIDKey),
|
||||
HostID: w.Header().Get(responseDeploymentIDKey),
|
||||
RequestID: w.Header().Get(xhttp.AmzRequestID),
|
||||
HostID: globalDeploymentID,
|
||||
})
|
||||
}
|
||||
if !started {
|
||||
setCommonHeaders(w)
|
||||
w.Header().Set("Content-Type", string(mimeJSON))
|
||||
w.Header().Set(xhttp.ContentType, string(mimeJSON))
|
||||
w.WriteHeader(hr.apiErr.HTTPStatusCode)
|
||||
}
|
||||
w.Write(errorRespJSON)
|
||||
@@ -785,6 +785,49 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
keepConnLive(w, respCh)
|
||||
}
|
||||
|
||||
func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "HealBackgroundStatus")
|
||||
|
||||
objectAPI := validateAdminReq(ctx, w, r)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Check if this setup has an erasure coded backend.
|
||||
if !globalIsXL {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrHealNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
var bgHealStates []madmin.BgHealState
|
||||
|
||||
// Get local heal status first
|
||||
bgHealStates = append(bgHealStates, getLocalBackgroundHealStatus())
|
||||
|
||||
if globalIsDistXL {
|
||||
// Get heal status from other peers
|
||||
peersHealStates := globalNotificationSys.BackgroundHealStatus()
|
||||
bgHealStates = append(bgHealStates, peersHealStates...)
|
||||
}
|
||||
|
||||
// Aggregate healing result
|
||||
var aggregatedHealStateResult = madmin.BgHealState{}
|
||||
for _, state := range bgHealStates {
|
||||
aggregatedHealStateResult.ScannedItemsCount += state.ScannedItemsCount
|
||||
if aggregatedHealStateResult.LastHealActivity.Before(state.LastHealActivity) {
|
||||
aggregatedHealStateResult.LastHealActivity = state.LastHealActivity
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if err := json.NewEncoder(w).Encode(aggregatedHealStateResult); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
w.(http.Flusher).Flush()
|
||||
}
|
||||
|
||||
// GetConfigHandler - GET /minio/admin/v1/config
|
||||
// Get config.json of this minio setup.
|
||||
func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -940,6 +983,12 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
vars := mux.Vars(r)
|
||||
accessKey := vars["accessKey"]
|
||||
|
||||
if err := globalIAMSys.DeleteUser(accessKey); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Notify all other MinIO peers to delete user.
|
||||
for _, nerr := range globalNotificationSys.DeleteUser(accessKey) {
|
||||
if nerr.Err != nil {
|
||||
@@ -1426,26 +1475,49 @@ func (a adminAPIHandlers) SetConfigKeysHandler(w http.ResponseWriter, r *http.Re
|
||||
func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "HTTPTrace")
|
||||
trcAll := r.URL.Query().Get("all") == "true"
|
||||
objectAPI := validateAdminReq(ctx, w, r)
|
||||
if objectAPI == nil {
|
||||
|
||||
// Validate request signature.
|
||||
adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Avoid reusing tcp connection if read timeout is hit
|
||||
// This is needed to make r.Context().Done() work as
|
||||
// expected in case of read timeout
|
||||
w.Header().Add("Connection", "close")
|
||||
w.Header().Add(xhttp.Connection, "close")
|
||||
|
||||
doneCh := make(chan struct{})
|
||||
defer close(doneCh)
|
||||
|
||||
traceCh := globalTrace.Trace(doneCh, trcAll)
|
||||
// Trace Publisher and peer-trace-client uses nonblocking send and hence does not wait for slow receivers.
|
||||
// Use buffered channel to take care of burst sends or slow w.Write()
|
||||
traceCh := make(chan interface{}, 4000)
|
||||
|
||||
filter := func(entry interface{}) bool {
|
||||
if trcAll {
|
||||
return true
|
||||
}
|
||||
trcInfo := entry.(trace.Info)
|
||||
return !strings.HasPrefix(trcInfo.ReqInfo.Path, minioReservedBucketPath)
|
||||
}
|
||||
remoteHosts := getRemoteHosts(globalEndpoints)
|
||||
peers, err := getRestClients(remoteHosts)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
globalHTTPTrace.Subscribe(traceCh, doneCh, filter)
|
||||
|
||||
for _, peer := range peers {
|
||||
peer.Trace(traceCh, doneCh, trcAll)
|
||||
}
|
||||
|
||||
enc := json.NewEncoder(w)
|
||||
for {
|
||||
select {
|
||||
case entry := <-traceCh:
|
||||
if _, err := w.Write(entry); err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := w.Write([]byte("\n")); err != nil {
|
||||
if err := enc.Encode(entry); err != nil {
|
||||
return
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
|
||||
@@ -194,7 +194,9 @@ var (
|
||||
"webhook": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"endpoint": ""
|
||||
"endpoint": "",
|
||||
"queueDir": "",
|
||||
"queueLimit": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -704,9 +706,6 @@ func TestAdminServerInfo(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, serverInfo := range results {
|
||||
if len(serverInfo.Addr) == 0 {
|
||||
t.Error("Expected server address to be non empty")
|
||||
}
|
||||
if serverInfo.Error != "" {
|
||||
t.Errorf("Unexpected error = %v\n", serverInfo.Error)
|
||||
}
|
||||
|
||||
+28
-3
@@ -131,6 +131,19 @@ func (ahs *allHealState) periodicHealSeqsClean() {
|
||||
}
|
||||
}
|
||||
|
||||
// getHealSequenceByToken - Retrieve a heal sequence by token. The second
|
||||
// argument returns if a heal sequence actually exists.
|
||||
func (ahs *allHealState) getHealSequenceByToken(token string) (h *healSequence, exists bool) {
|
||||
ahs.Lock()
|
||||
defer ahs.Unlock()
|
||||
for _, healSeq := range ahs.healSeqMap {
|
||||
if healSeq.clientToken == token {
|
||||
return healSeq, true
|
||||
}
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// getHealSequence - Retrieve a heal sequence by path. The second
|
||||
// argument returns if a heal sequence actually exists.
|
||||
func (ahs *allHealState) getHealSequence(path string) (h *healSequence, exists bool) {
|
||||
@@ -335,6 +348,12 @@ type healSequence struct {
|
||||
// the last result index sent to client
|
||||
lastSentResultIndex int64
|
||||
|
||||
// Number of total items scanned
|
||||
scannedItemsCount int64
|
||||
|
||||
// The time of the last scan/heal activity
|
||||
lastHealActivity time.Time
|
||||
|
||||
// Holds the request-info for logging
|
||||
ctx context.Context
|
||||
}
|
||||
@@ -552,17 +571,20 @@ func (h *healSequence) queueHealTask(path string, healType madmin.HealItemType)
|
||||
}
|
||||
|
||||
func (h *healSequence) healItemsFromSourceCh() error {
|
||||
h.lastHealActivity = UTCNow()
|
||||
|
||||
// Start healing the config prefix.
|
||||
if err := h.healMinioSysMeta(minioConfigPrefix)(); err != nil {
|
||||
return err
|
||||
logger.LogIf(h.ctx, err)
|
||||
}
|
||||
|
||||
// Start healing the bucket config prefix.
|
||||
if err := h.healMinioSysMeta(bucketConfigPrefix)(); err != nil {
|
||||
return err
|
||||
logger.LogIf(h.ctx, err)
|
||||
}
|
||||
|
||||
for path := range h.sourceCh {
|
||||
|
||||
var itemType madmin.HealItemType
|
||||
switch {
|
||||
case path == "/":
|
||||
@@ -574,8 +596,11 @@ func (h *healSequence) healItemsFromSourceCh() error {
|
||||
}
|
||||
|
||||
if err := h.queueHealTask(path, itemType); err != nil {
|
||||
return err
|
||||
logger.LogIf(h.ctx, err)
|
||||
}
|
||||
|
||||
h.scannedItemsCount++
|
||||
h.lastHealActivity = UTCNow()
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -61,6 +61,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
|
||||
adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
|
||||
adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
|
||||
|
||||
adminV1Router.Methods(http.MethodPost).Path("/background-heal/status").HandlerFunc(httpTraceAll(adminAPI.BackgroundHealStatusHandler))
|
||||
|
||||
/// Health operations
|
||||
|
||||
}
|
||||
|
||||
@@ -20,13 +20,6 @@ import (
|
||||
"encoding/xml"
|
||||
)
|
||||
|
||||
const (
|
||||
// Response request id.
|
||||
responseRequestIDKey = "x-amz-request-id"
|
||||
// Deployment id.
|
||||
responseDeploymentIDKey = "x-minio-deployment-id"
|
||||
)
|
||||
|
||||
// ObjectIdentifier carries key name for the object to delete.
|
||||
type ObjectIdentifier struct {
|
||||
ObjectName string `xml:"Key"`
|
||||
|
||||
+11
-10
@@ -26,6 +26,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
)
|
||||
|
||||
// Returns a hexadecimal representation of time at the
|
||||
@@ -36,13 +37,13 @@ func mustGetRequestID(t time.Time) string {
|
||||
|
||||
// Write http common headers
|
||||
func setCommonHeaders(w http.ResponseWriter) {
|
||||
w.Header().Set("Server", "MinIO/"+ReleaseTag)
|
||||
w.Header().Set(xhttp.ServerInfo, "MinIO/"+ReleaseTag)
|
||||
// Set `x-amz-bucket-region` only if region is set on the server
|
||||
// by default minio uses an empty region.
|
||||
if region := globalServerConfig.GetRegion(); region != "" {
|
||||
w.Header().Set("X-Amz-Bucket-Region", region)
|
||||
w.Header().Set(xhttp.AmzBucketRegion, region)
|
||||
}
|
||||
w.Header().Set("Accept-Ranges", "bytes")
|
||||
w.Header().Set(xhttp.AcceptRanges, "bytes")
|
||||
|
||||
// Remove sensitive information
|
||||
crypto.RemoveSensitiveHeaders(w.Header())
|
||||
@@ -72,23 +73,23 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
|
||||
|
||||
// Set last modified time.
|
||||
lastModified := objInfo.ModTime.UTC().Format(http.TimeFormat)
|
||||
w.Header().Set("Last-Modified", lastModified)
|
||||
w.Header().Set(xhttp.LastModified, lastModified)
|
||||
|
||||
// Set Etag if available.
|
||||
if objInfo.ETag != "" {
|
||||
w.Header()["ETag"] = []string{"\"" + objInfo.ETag + "\""}
|
||||
w.Header()[xhttp.ETag] = []string{"\"" + objInfo.ETag + "\""}
|
||||
}
|
||||
|
||||
if objInfo.ContentType != "" {
|
||||
w.Header().Set("Content-Type", objInfo.ContentType)
|
||||
w.Header().Set(xhttp.ContentType, objInfo.ContentType)
|
||||
}
|
||||
|
||||
if objInfo.ContentEncoding != "" {
|
||||
w.Header().Set("Content-Encoding", objInfo.ContentEncoding)
|
||||
w.Header().Set(xhttp.ContentEncoding, objInfo.ContentEncoding)
|
||||
}
|
||||
|
||||
if !objInfo.Expires.IsZero() {
|
||||
w.Header().Set("Expires", objInfo.Expires.UTC().Format(http.TimeFormat))
|
||||
w.Header().Set(xhttp.Expires, objInfo.Expires.UTC().Format(http.TimeFormat))
|
||||
}
|
||||
|
||||
// Set all other user defined metadata.
|
||||
@@ -124,10 +125,10 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
|
||||
}
|
||||
|
||||
// Set content length.
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(rangeLen, 10))
|
||||
w.Header().Set(xhttp.ContentLength, strconv.FormatInt(rangeLen, 10))
|
||||
if rs != nil {
|
||||
contentRange := fmt.Sprintf("bytes %d-%d/%d", start, start+rangeLen-1, totalObjectSize)
|
||||
w.Header().Set("Content-Range", contentRange)
|
||||
w.Header().Set(xhttp.ContentRange, contentRange)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
+14
-13
@@ -26,6 +26,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/handlers"
|
||||
)
|
||||
@@ -522,9 +523,9 @@ func generateMultiDeleteResponse(quiet bool, deletedObjects []ObjectIdentifier,
|
||||
func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) {
|
||||
setCommonHeaders(w)
|
||||
if mType != mimeNone {
|
||||
w.Header().Set("Content-Type", string(mType))
|
||||
w.Header().Set(xhttp.ContentType, string(mType))
|
||||
}
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(response)))
|
||||
w.Header().Set(xhttp.ContentLength, strconv.Itoa(len(response)))
|
||||
w.WriteHeader(statusCode)
|
||||
if response != nil {
|
||||
w.Write(response)
|
||||
@@ -563,7 +564,7 @@ func writeSuccessNoContent(w http.ResponseWriter) {
|
||||
|
||||
// writeRedirectSeeOther writes Location header with http status 303
|
||||
func writeRedirectSeeOther(w http.ResponseWriter, location string) {
|
||||
w.Header().Set("Location", location)
|
||||
w.Header().Set(xhttp.Location, location)
|
||||
writeResponse(w, http.StatusSeeOther, nil, mimeNone)
|
||||
}
|
||||
|
||||
@@ -577,12 +578,12 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
|
||||
case "SlowDown", "XMinioServerNotInitialized", "XMinioReadQuorum", "XMinioWriteQuorum":
|
||||
// Set retry-after header to indicate user-agents to retry request after 120secs.
|
||||
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
|
||||
w.Header().Set("Retry-After", "120")
|
||||
w.Header().Set(xhttp.RetryAfter, "120")
|
||||
case "AccessDenied":
|
||||
// The request is from browser and also if browser
|
||||
// is enabled we need to redirect.
|
||||
if browser {
|
||||
w.Header().Set("Location", minioReservedBucketPath+reqURL.Path)
|
||||
w.Header().Set(xhttp.Location, minioReservedBucketPath+reqURL.Path)
|
||||
w.WriteHeader(http.StatusTemporaryRedirect)
|
||||
return
|
||||
}
|
||||
@@ -590,7 +591,7 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
|
||||
|
||||
// Generate error response.
|
||||
errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path,
|
||||
w.Header().Get(responseRequestIDKey), w.Header().Get(responseDeploymentIDKey))
|
||||
w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
|
||||
encodedErrorResponse := encodeResponse(errorResponse)
|
||||
writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeXML)
|
||||
}
|
||||
@@ -603,7 +604,7 @@ func writeErrorResponseHeadersOnly(w http.ResponseWriter, err APIError) {
|
||||
// useful for admin APIs.
|
||||
func writeErrorResponseJSON(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL) {
|
||||
// Generate error response.
|
||||
errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path, w.Header().Get(responseRequestIDKey), w.Header().Get(responseDeploymentIDKey))
|
||||
errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path, w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
|
||||
encodedErrorResponse := encodeResponseJSON(errorResponse)
|
||||
writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
|
||||
}
|
||||
@@ -621,8 +622,8 @@ func writeCustomErrorResponseJSON(ctx context.Context, w http.ResponseWriter, er
|
||||
Resource: reqURL.Path,
|
||||
BucketName: reqInfo.BucketName,
|
||||
Key: reqInfo.ObjectName,
|
||||
RequestID: w.Header().Get(responseRequestIDKey),
|
||||
HostID: w.Header().Get(responseDeploymentIDKey),
|
||||
RequestID: w.Header().Get(xhttp.AmzRequestID),
|
||||
HostID: globalDeploymentID,
|
||||
}
|
||||
encodedErrorResponse := encodeResponseJSON(errorResponse)
|
||||
writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
|
||||
@@ -637,12 +638,12 @@ func writeCustomErrorResponseXML(ctx context.Context, w http.ResponseWriter, err
|
||||
case "SlowDown", "XMinioServerNotInitialized", "XMinioReadQuorum", "XMinioWriteQuorum":
|
||||
// Set retry-after header to indicate user-agents to retry request after 120secs.
|
||||
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
|
||||
w.Header().Set("Retry-After", "120")
|
||||
w.Header().Set(xhttp.RetryAfter, "120")
|
||||
case "AccessDenied":
|
||||
// The request is from browser and also if browser
|
||||
// is enabled we need to redirect.
|
||||
if browser && globalIsBrowserEnabled {
|
||||
w.Header().Set("Location", minioReservedBucketPath+reqURL.Path)
|
||||
w.Header().Set(xhttp.Location, minioReservedBucketPath+reqURL.Path)
|
||||
w.WriteHeader(http.StatusTemporaryRedirect)
|
||||
return
|
||||
}
|
||||
@@ -655,8 +656,8 @@ func writeCustomErrorResponseXML(ctx context.Context, w http.ResponseWriter, err
|
||||
Resource: reqURL.Path,
|
||||
BucketName: reqInfo.BucketName,
|
||||
Key: reqInfo.ObjectName,
|
||||
RequestID: w.Header().Get(responseRequestIDKey),
|
||||
HostID: w.Header().Get(responseDeploymentIDKey),
|
||||
RequestID: w.Header().Get(xhttp.AmzRequestID),
|
||||
HostID: globalDeploymentID,
|
||||
}
|
||||
|
||||
encodedErrorResponse := encodeResponse(errorResponse)
|
||||
|
||||
+49
-43
@@ -20,6 +20,7 @@ import (
|
||||
"net/http"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
)
|
||||
|
||||
// objectAPIHandler implements and provides http handlers for S3 API.
|
||||
@@ -28,10 +29,12 @@ type objectAPIHandlers struct {
|
||||
CacheAPI func() CacheObjectLayer
|
||||
// Returns true of handlers should interpret encryption.
|
||||
EncryptionEnabled func() bool
|
||||
// Returns true if handlers allow SSE-KMS encryption headers.
|
||||
AllowSSEKMS func() bool
|
||||
}
|
||||
|
||||
// registerAPIRouter - registers S3 compatible APIs.
|
||||
func registerAPIRouter(router *mux.Router, encryptionEnabled bool) {
|
||||
func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool) {
|
||||
// Initialize API.
|
||||
api := objectAPIHandlers{
|
||||
ObjectAPI: newObjectLayerFn,
|
||||
@@ -39,6 +42,9 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled bool) {
|
||||
EncryptionEnabled: func() bool {
|
||||
return encryptionEnabled
|
||||
},
|
||||
AllowSSEKMS: func() bool {
|
||||
return allowSSEKMS
|
||||
},
|
||||
}
|
||||
|
||||
// API Router
|
||||
@@ -53,98 +59,98 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled bool) {
|
||||
for _, bucket := range routers {
|
||||
// Object operations
|
||||
// HeadObject
|
||||
bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.HeadObjectHandler))
|
||||
bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.HeadObjectHandler))
|
||||
// CopyObjectPart
|
||||
bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
|
||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
|
||||
// PutObjectPart
|
||||
bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
|
||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
|
||||
// ListObjectPxarts
|
||||
bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.ListObjectPartsHandler)).Queries("uploadId", "{uploadId:.*}")
|
||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.ListObjectPartsHandler)).Queries("uploadId", "{uploadId:.*}")
|
||||
// CompleteMultipartUpload
|
||||
bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.CompleteMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
|
||||
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.CompleteMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
|
||||
// NewMultipartUpload
|
||||
bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.NewMultipartUploadHandler)).Queries("uploads", "")
|
||||
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.NewMultipartUploadHandler)).Queries("uploads", "")
|
||||
// AbortMultipartUpload
|
||||
bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
|
||||
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
|
||||
// GetObjectACL - this is a dummy call.
|
||||
bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectACLHandler)).Queries("acl", "")
|
||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectACLHandler)).Queries("acl", "")
|
||||
// GetObjectTagging - this is a dummy call.
|
||||
bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectTaggingHandler)).Queries("tagging", "")
|
||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectTaggingHandler)).Queries("tagging", "")
|
||||
// SelectObjectContent
|
||||
bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.SelectObjectContentHandler)).Queries("select", "").Queries("select-type", "2")
|
||||
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.SelectObjectContentHandler)).Queries("select", "").Queries("select-type", "2")
|
||||
// GetObject
|
||||
bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler))
|
||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler))
|
||||
// CopyObject
|
||||
bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectHandler))
|
||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectHandler))
|
||||
// PutObject
|
||||
bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectHandler))
|
||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectHandler))
|
||||
// DeleteObject
|
||||
bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.DeleteObjectHandler))
|
||||
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.DeleteObjectHandler))
|
||||
|
||||
/// Bucket operations
|
||||
// GetBucketLocation
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLocationHandler)).Queries("location", "")
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketLocationHandler)).Queries("location", "")
|
||||
// GetBucketPolicy
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "")
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "")
|
||||
|
||||
// Dummy Bucket Calls
|
||||
// GetBucketACL -- this is a dummy call.
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketACLHandler)).Queries("acl", "")
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketACLHandler)).Queries("acl", "")
|
||||
// GetBucketCors - this is a dummy call.
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketCorsHandler)).Queries("cors", "")
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketCorsHandler)).Queries("cors", "")
|
||||
// GetBucketWebsiteHandler - this is a dummy call.
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketWebsiteHandler)).Queries("website", "")
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketWebsiteHandler)).Queries("website", "")
|
||||
// GetBucketVersioningHandler - this is a dummy call.
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketVersioningHandler)).Queries("versioning", "")
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketVersioningHandler)).Queries("versioning", "")
|
||||
// GetBucketAccelerateHandler - this is a dummy call.
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketAccelerateHandler)).Queries("accelerate", "")
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketAccelerateHandler)).Queries("accelerate", "")
|
||||
// GetBucketRequestPaymentHandler - this is a dummy call.
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketRequestPaymentHandler)).Queries("requestPayment", "")
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketRequestPaymentHandler)).Queries("requestPayment", "")
|
||||
// GetBucketLoggingHandler - this is a dummy call.
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLoggingHandler)).Queries("logging", "")
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketLoggingHandler)).Queries("logging", "")
|
||||
// GetBucketLifecycleHandler - this is a dummy call.
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLifecycleHandler)).Queries("lifecycle", "")
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketLifecycleHandler)).Queries("lifecycle", "")
|
||||
// GetBucketReplicationHandler - this is a dummy call.
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketReplicationHandler)).Queries("replication", "")
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketReplicationHandler)).Queries("replication", "")
|
||||
// GetBucketTaggingHandler - this is a dummy call.
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketTaggingHandler)).Queries("tagging", "")
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketTaggingHandler)).Queries("tagging", "")
|
||||
//DeleteBucketWebsiteHandler
|
||||
bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketWebsiteHandler)).Queries("website", "")
|
||||
bucket.Methods(http.MethodDelete).HandlerFunc(httpTraceAll(api.DeleteBucketWebsiteHandler)).Queries("website", "")
|
||||
// DeleteBucketTaggingHandler
|
||||
bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketTaggingHandler)).Queries("tagging", "")
|
||||
bucket.Methods(http.MethodDelete).HandlerFunc(httpTraceAll(api.DeleteBucketTaggingHandler)).Queries("tagging", "")
|
||||
|
||||
// GetBucketNotification
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "")
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "")
|
||||
// ListenBucketNotification
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListenBucketNotificationHandler)).Queries("events", "{events:.*}")
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListenBucketNotificationHandler)).Queries("events", "{events:.*}")
|
||||
// ListMultipartUploads
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListMultipartUploadsHandler)).Queries("uploads", "")
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListMultipartUploadsHandler)).Queries("uploads", "")
|
||||
// ListObjectsV2
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV2Handler)).Queries("list-type", "2")
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListObjectsV2Handler)).Queries("list-type", "2")
|
||||
// ListObjectsV1 (Legacy)
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV1Handler))
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListObjectsV1Handler))
|
||||
// PutBucketPolicy
|
||||
bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketPolicyHandler)).Queries("policy", "")
|
||||
bucket.Methods(http.MethodPut).HandlerFunc(httpTraceAll(api.PutBucketPolicyHandler)).Queries("policy", "")
|
||||
// PutBucketNotification
|
||||
bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketNotificationHandler)).Queries("notification", "")
|
||||
bucket.Methods(http.MethodPut).HandlerFunc(httpTraceAll(api.PutBucketNotificationHandler)).Queries("notification", "")
|
||||
// PutBucket
|
||||
bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketHandler))
|
||||
bucket.Methods(http.MethodPut).HandlerFunc(httpTraceAll(api.PutBucketHandler))
|
||||
// HeadBucket
|
||||
bucket.Methods("HEAD").HandlerFunc(httpTraceAll(api.HeadBucketHandler))
|
||||
bucket.Methods(http.MethodHead).HandlerFunc(httpTraceAll(api.HeadBucketHandler))
|
||||
// PostPolicy
|
||||
bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceHdrs(api.PostPolicyBucketHandler))
|
||||
bucket.Methods(http.MethodPost).HeadersRegexp(xhttp.ContentType, "multipart/form-data*").HandlerFunc(httpTraceHdrs(api.PostPolicyBucketHandler))
|
||||
// DeleteMultipleObjects
|
||||
bucket.Methods("POST").HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler)).Queries("delete", "")
|
||||
bucket.Methods(http.MethodPost).HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler)).Queries("delete", "")
|
||||
// DeleteBucketPolicy
|
||||
bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketPolicyHandler)).Queries("policy", "")
|
||||
bucket.Methods(http.MethodDelete).HandlerFunc(httpTraceAll(api.DeleteBucketPolicyHandler)).Queries("policy", "")
|
||||
// DeleteBucket
|
||||
bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketHandler))
|
||||
bucket.Methods(http.MethodDelete).HandlerFunc(httpTraceAll(api.DeleteBucketHandler))
|
||||
}
|
||||
|
||||
/// Root operation
|
||||
|
||||
// ListBuckets
|
||||
apiRouter.Methods("GET").Path("/").HandlerFunc(httpTraceAll(api.ListBucketsHandler))
|
||||
apiRouter.Methods(http.MethodGet).Path("/").HandlerFunc(httpTraceAll(api.ListBucketsHandler))
|
||||
|
||||
// If none of the routes match.
|
||||
apiRouter.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler))
|
||||
|
||||
+54
-19
@@ -29,6 +29,7 @@ import (
|
||||
"strings"
|
||||
|
||||
jwtgo "github.com/dgrijalva/jwt-go"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
@@ -38,41 +39,41 @@ import (
|
||||
|
||||
// Verify if request has JWT.
|
||||
func isRequestJWT(r *http.Request) bool {
|
||||
return strings.HasPrefix(r.Header.Get("Authorization"), jwtAlgorithm)
|
||||
return strings.HasPrefix(r.Header.Get(xhttp.Authorization), jwtAlgorithm)
|
||||
}
|
||||
|
||||
// Verify if request has AWS Signature Version '4'.
|
||||
func isRequestSignatureV4(r *http.Request) bool {
|
||||
return strings.HasPrefix(r.Header.Get("Authorization"), signV4Algorithm)
|
||||
return strings.HasPrefix(r.Header.Get(xhttp.Authorization), signV4Algorithm)
|
||||
}
|
||||
|
||||
// Verify if request has AWS Signature Version '2'.
|
||||
func isRequestSignatureV2(r *http.Request) bool {
|
||||
return (!strings.HasPrefix(r.Header.Get("Authorization"), signV4Algorithm) &&
|
||||
strings.HasPrefix(r.Header.Get("Authorization"), signV2Algorithm))
|
||||
return (!strings.HasPrefix(r.Header.Get(xhttp.Authorization), signV4Algorithm) &&
|
||||
strings.HasPrefix(r.Header.Get(xhttp.Authorization), signV2Algorithm))
|
||||
}
|
||||
|
||||
// Verify if request has AWS PreSign Version '4'.
|
||||
func isRequestPresignedSignatureV4(r *http.Request) bool {
|
||||
_, ok := r.URL.Query()["X-Amz-Credential"]
|
||||
_, ok := r.URL.Query()[xhttp.AmzCredential]
|
||||
return ok
|
||||
}
|
||||
|
||||
// Verify request has AWS PreSign Version '2'.
|
||||
func isRequestPresignedSignatureV2(r *http.Request) bool {
|
||||
_, ok := r.URL.Query()["AWSAccessKeyId"]
|
||||
_, ok := r.URL.Query()[xhttp.AmzAccessKeyID]
|
||||
return ok
|
||||
}
|
||||
|
||||
// Verify if request has AWS Post policy Signature Version '4'.
|
||||
func isRequestPostPolicySignatureV4(r *http.Request) bool {
|
||||
return strings.Contains(r.Header.Get("Content-Type"), "multipart/form-data") &&
|
||||
return strings.Contains(r.Header.Get(xhttp.ContentType), "multipart/form-data") &&
|
||||
r.Method == http.MethodPost
|
||||
}
|
||||
|
||||
// Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation.
|
||||
func isRequestSignStreamingV4(r *http.Request) bool {
|
||||
return r.Header.Get("x-amz-content-sha256") == streamingContentSHA256 &&
|
||||
return r.Header.Get(xhttp.AmzContentSha256) == streamingContentSHA256 &&
|
||||
r.Method == http.MethodPut
|
||||
}
|
||||
|
||||
@@ -109,9 +110,9 @@ func getRequestAuthType(r *http.Request) authType {
|
||||
return authTypeJWT
|
||||
} else if isRequestPostPolicySignatureV4(r) {
|
||||
return authTypePostPolicy
|
||||
} else if _, ok := r.URL.Query()["Action"]; ok {
|
||||
} else if _, ok := r.URL.Query()[xhttp.Action]; ok {
|
||||
return authTypeSTS
|
||||
} else if _, ok := r.Header["Authorization"]; !ok {
|
||||
} else if _, ok := r.Header[xhttp.Authorization]; !ok {
|
||||
return authTypeAnonymous
|
||||
}
|
||||
return authTypeUnknown
|
||||
@@ -121,7 +122,7 @@ func getRequestAuthType(r *http.Request) authType {
|
||||
// It does not accept presigned or JWT or anonymous requests.
|
||||
func checkAdminRequestAuthType(ctx context.Context, r *http.Request, region string) APIErrorCode {
|
||||
s3Err := ErrAccessDenied
|
||||
if _, ok := r.Header["X-Amz-Content-Sha256"]; ok &&
|
||||
if _, ok := r.Header[xhttp.AmzContentSha256]; ok &&
|
||||
getRequestAuthType(r) == authTypeSigned && !skipContentSha256Cksum(r) {
|
||||
// We only support admin credentials to access admin APIs.
|
||||
|
||||
@@ -148,11 +149,11 @@ func checkAdminRequestAuthType(ctx context.Context, r *http.Request, region stri
|
||||
|
||||
// Fetch the security token set by the client.
|
||||
func getSessionToken(r *http.Request) (token string) {
|
||||
token = r.Header.Get("X-Amz-Security-Token")
|
||||
token = r.Header.Get(xhttp.AmzSecurityToken)
|
||||
if token != "" {
|
||||
return token
|
||||
}
|
||||
return r.URL.Query().Get("X-Amz-Security-Token")
|
||||
return r.URL.Query().Get(xhttp.AmzSecurityToken)
|
||||
}
|
||||
|
||||
// Fetch claims in the security token returned by the client, doesn't return
|
||||
@@ -200,6 +201,39 @@ func getClaimsFromToken(r *http.Request) (map[string]interface{}, error) {
|
||||
if _, ok = v.(string); !ok {
|
||||
return nil, errInvalidAccessKeyID
|
||||
}
|
||||
|
||||
if globalPolicyOPA == nil {
|
||||
// If OPA is not set, session token should
|
||||
// have a policy and its mandatory, reject
|
||||
// requests without policy claim.
|
||||
p, pok := claims[iampolicy.PolicyName]
|
||||
if !pok {
|
||||
return nil, errAuthentication
|
||||
}
|
||||
if _, pok = p.(string); !pok {
|
||||
return nil, errAuthentication
|
||||
}
|
||||
sp, spok := claims[iampolicy.SessionPolicyName]
|
||||
// Sub policy is optional, if not set return success.
|
||||
if !spok {
|
||||
return claims, nil
|
||||
}
|
||||
// Sub policy is set but its not a string, reject such requests
|
||||
spStr, spok := sp.(string)
|
||||
if !spok {
|
||||
return nil, errAuthentication
|
||||
}
|
||||
// Looks like subpolicy is set and is a string, if set then its
|
||||
// base64 encoded, decode it. Decoding fails reject such requests.
|
||||
spBytes, err := base64.StdEncoding.DecodeString(spStr)
|
||||
if err != nil {
|
||||
// Base64 decoding fails, we should log to indicate
|
||||
// something is malforming the request sent by client.
|
||||
logger.LogIf(context.Background(), err)
|
||||
return nil, errAuthentication
|
||||
}
|
||||
claims[iampolicy.SessionPolicyName] = string(spBytes)
|
||||
}
|
||||
return claims, nil
|
||||
}
|
||||
|
||||
@@ -337,8 +371,8 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
|
||||
contentMD5, contentSHA256 []byte
|
||||
)
|
||||
// Extract 'Content-Md5' if present.
|
||||
if _, ok := r.Header["Content-Md5"]; ok {
|
||||
contentMD5, err = base64.StdEncoding.Strict().DecodeString(r.Header.Get("Content-Md5"))
|
||||
if _, ok := r.Header[xhttp.ContentMD5]; ok {
|
||||
contentMD5, err = base64.StdEncoding.Strict().DecodeString(r.Header.Get(xhttp.ContentMD5))
|
||||
if err != nil || len(contentMD5) == 0 {
|
||||
return ErrInvalidDigest
|
||||
}
|
||||
@@ -347,14 +381,14 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
|
||||
// Extract either 'X-Amz-Content-Sha256' header or 'X-Amz-Content-Sha256' query parameter (if V4 presigned)
|
||||
// Do not verify 'X-Amz-Content-Sha256' if skipSHA256.
|
||||
if skipSHA256 := skipContentSha256Cksum(r); !skipSHA256 && isRequestPresignedSignatureV4(r) {
|
||||
if sha256Sum, ok := r.URL.Query()["X-Amz-Content-Sha256"]; ok && len(sha256Sum) > 0 {
|
||||
if sha256Sum, ok := r.URL.Query()[xhttp.AmzContentSha256]; ok && len(sha256Sum) > 0 {
|
||||
contentSHA256, err = hex.DecodeString(sha256Sum[0])
|
||||
if err != nil {
|
||||
return ErrContentSHA256Mismatch
|
||||
}
|
||||
}
|
||||
} else if _, ok := r.Header["X-Amz-Content-Sha256"]; !skipSHA256 && ok {
|
||||
contentSHA256, err = hex.DecodeString(r.Header.Get("X-Amz-Content-Sha256"))
|
||||
} else if _, ok := r.Header[xhttp.AmzContentSha256]; !skipSHA256 && ok {
|
||||
contentSHA256, err = hex.DecodeString(r.Header.Get(xhttp.AmzContentSha256))
|
||||
if err != nil || len(contentSHA256) == 0 {
|
||||
return ErrContentSHA256Mismatch
|
||||
}
|
||||
@@ -362,7 +396,8 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
|
||||
|
||||
// Verify 'Content-Md5' and/or 'X-Amz-Content-Sha256' if present.
|
||||
// The verification happens implicit during reading.
|
||||
reader, err := hash.NewReader(r.Body, -1, hex.EncodeToString(contentMD5), hex.EncodeToString(contentSHA256), -1, globalCLIContext.StrictS3Compat)
|
||||
reader, err := hash.NewReader(r.Body, -1, hex.EncodeToString(contentMD5),
|
||||
hex.EncodeToString(contentSHA256), -1, globalCLIContext.StrictS3Compat)
|
||||
if err != nil {
|
||||
return toAPIErrorCode(ctx, err)
|
||||
}
|
||||
|
||||
@@ -131,7 +131,7 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
|
||||
b.h.Write(buf)
|
||||
|
||||
if !bytes.Equal(b.h.Sum(nil), b.hashBytes) {
|
||||
err = hashMismatchError{hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil))}
|
||||
err = HashMismatchError{hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil))}
|
||||
logger.LogIf(context.Background(), err)
|
||||
return 0, err
|
||||
}
|
||||
|
||||
@@ -155,33 +155,3 @@ func bitrotWriterSum(w io.Writer) []byte {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Verify if a file has bitrot error.
|
||||
func bitrotCheckFile(disk StorageAPI, volume string, filePath string, tillOffset int64, algo BitrotAlgorithm, sum []byte, shardSize int64) (err error) {
|
||||
if algo != HighwayHash256S {
|
||||
buf := []byte{}
|
||||
// For whole-file bitrot we don't need to read the entire file as the bitrot verify happens on the server side even if we read 0-bytes.
|
||||
_, err = disk.ReadFile(volume, filePath, 0, buf, NewBitrotVerifier(algo, sum))
|
||||
return err
|
||||
}
|
||||
buf := make([]byte, shardSize)
|
||||
r := newStreamingBitrotReader(disk, volume, filePath, tillOffset, algo, shardSize)
|
||||
defer closeBitrotReaders([]io.ReaderAt{r})
|
||||
var offset int64
|
||||
for {
|
||||
if offset == tillOffset {
|
||||
break
|
||||
}
|
||||
var n int
|
||||
tmpBuf := buf
|
||||
if int64(len(tmpBuf)) > (tillOffset - offset) {
|
||||
tmpBuf = tmpBuf[:(tillOffset - offset)]
|
||||
}
|
||||
n, err = r.ReadAt(tmpBuf, offset)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
offset += int64(n)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -46,13 +46,6 @@ func validateListObjectsArgs(prefix, marker, delimiter, encodingType string, max
|
||||
}
|
||||
}
|
||||
|
||||
/// MinIO special conditions for ListObjects.
|
||||
|
||||
// Verify if delimiter is anything other than '/', which we do not support.
|
||||
if delimiter != "" && delimiter != "/" {
|
||||
return ErrNotImplemented
|
||||
}
|
||||
// Success.
|
||||
return ErrNone
|
||||
}
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ import (
|
||||
|
||||
"github.com/minio/minio-go/v6/pkg/set"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/dns"
|
||||
"github.com/minio/minio/pkg/event"
|
||||
@@ -445,7 +446,8 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
|
||||
// Make sure to add Location information here only for bucket
|
||||
w.Header().Set("Location", getObjectLocation(r, globalDomainNames, bucket, ""))
|
||||
w.Header().Set(xhttp.Location,
|
||||
getObjectLocation(r, globalDomainNames, bucket, ""))
|
||||
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
return
|
||||
@@ -466,7 +468,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
|
||||
// Make sure to add Location information here only for bucket
|
||||
w.Header().Set("Location", path.Clean(r.URL.Path)) // Clean any trailing slashes.
|
||||
w.Header().Set(xhttp.Location, path.Clean(r.URL.Path)) // Clean any trailing slashes.
|
||||
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
@@ -498,6 +500,9 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
|
||||
bucket := mux.Vars(r)["bucket"]
|
||||
|
||||
// To detect if the client has disconnected.
|
||||
r.Body = &detectDisconnect{r.Body, r.Context().Done()}
|
||||
|
||||
// Require Content-Length to be set in the request
|
||||
size := r.ContentLength
|
||||
if size < 0 {
|
||||
@@ -678,8 +683,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
}
|
||||
|
||||
location := getObjectLocation(r, globalDomainNames, bucket, object)
|
||||
w.Header()["ETag"] = []string{`"` + objInfo.ETag + `"`}
|
||||
w.Header().Set("Location", location)
|
||||
w.Header()[xhttp.ETag] = []string{`"` + objInfo.ETag + `"`}
|
||||
w.Header().Set(xhttp.Location, location)
|
||||
|
||||
// Notify object created event.
|
||||
defer sendEvent(eventArgs{
|
||||
|
||||
@@ -767,7 +767,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
|
||||
for id, args := range config.Notify.Webhook {
|
||||
if args.Enable {
|
||||
args.RootCAs = globalRootCAs
|
||||
newTarget := target.NewWebhookTarget(id, args)
|
||||
newTarget := target.NewWebhookTarget(id, args, GlobalServiceDoneCh)
|
||||
if err := targetList.Add(newTarget); err != nil {
|
||||
logger.LogIf(context.Background(), err)
|
||||
continue
|
||||
|
||||
@@ -203,7 +203,7 @@ func TestValidateConfig(t *testing.T) {
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "kafka": { "1": { "enable": true, "brokers": null, "topic": "", "queueDir": "", "queueLimit": 0 } }}}`, false},
|
||||
|
||||
// Test 17 - Test Webhook
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "webhook": { "1": { "enable": true, "endpoint": "" } }}}`, false},
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "webhook": { "1": { "enable": true, "endpoint": "", "queueDir": "", "queueLimit": 0} }}}`, false},
|
||||
|
||||
// Test 18 - Test MySQL
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
|
||||
|
||||
@@ -2440,7 +2440,7 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
|
||||
// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
|
||||
// and configFile, take a transaction lock to avoid data race between readConfig()
|
||||
// and saveConfig().
|
||||
objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
|
||||
objLock := globalNSMutex.NewNSLock(context.Background(), minioMetaBucket, transactionConfigFile)
|
||||
if err = objLock.GetLock(globalOperationTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -2492,7 +2492,7 @@ func migrateMinioSysConfig(objAPI ObjectLayer) error {
|
||||
// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
|
||||
// and configFile, take a transaction lock to avoid data race between readConfig()
|
||||
// and saveConfig().
|
||||
objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
|
||||
objLock := globalNSMutex.NewNSLock(context.Background(), minioMetaBucket, transactionConfigFile)
|
||||
if err := objLock.GetLock(globalOperationTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -884,7 +884,7 @@ type serverConfigV32 struct {
|
||||
} `json:"policy"`
|
||||
}
|
||||
|
||||
// serverConfigV33 is just like version '32', removes clientID from NATS and MQTT, and adds queueDir, queueLimit in MQTT and kafka.
|
||||
// serverConfigV33 is just like version '32', removes clientID from NATS and MQTT, and adds queueDir, queueLimit in notification targets.
|
||||
type serverConfigV33 struct {
|
||||
quick.Config `json:"-"` // ignore interfaces
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
@@ -125,6 +126,25 @@ func (s3KMS) IsRequested(h http.Header) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// ParseHTTP parses the SSE-KMS headers and returns the SSE-KMS key ID
|
||||
// and context, if present, on success.
|
||||
func (s3KMS) ParseHTTP(h http.Header) (string, interface{}, error) {
|
||||
algorithm := h.Get(SSEHeader)
|
||||
if algorithm != SSEAlgorithmKMS {
|
||||
return "", nil, ErrInvalidEncryptionMethod
|
||||
}
|
||||
|
||||
contextStr, ok := h[SSEKmsContext]
|
||||
if ok {
|
||||
var context map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(contextStr[0]), &context); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
return h.Get(SSEKmsID), context, nil
|
||||
}
|
||||
return h.Get(SSEKmsID), nil, nil
|
||||
}
|
||||
|
||||
var (
|
||||
// SSEC represents AWS SSE-C. It provides functionality to handle
|
||||
// SSE-C requests.
|
||||
|
||||
@@ -54,6 +54,56 @@ func TestKMSIsRequested(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
var kmsParseHTTPTests = []struct {
|
||||
Header http.Header
|
||||
ShouldFail bool
|
||||
}{
|
||||
{Header: http.Header{}, ShouldFail: true}, // 0
|
||||
{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"aws:kms"}}, ShouldFail: false}, // 1
|
||||
{Header: http.Header{
|
||||
"X-Amz-Server-Side-Encryption": []string{"aws:kms"},
|
||||
"X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": []string{"s3-007-293847485-724784"},
|
||||
}, ShouldFail: false}, // 2
|
||||
{Header: http.Header{
|
||||
"X-Amz-Server-Side-Encryption": []string{"aws:kms"},
|
||||
"X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": []string{"s3-007-293847485-724784"},
|
||||
"X-Amz-Server-Side-Encryption-Context": []string{"{}"},
|
||||
}, ShouldFail: false}, // 3
|
||||
{Header: http.Header{
|
||||
"X-Amz-Server-Side-Encryption": []string{"aws:kms"},
|
||||
"X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": []string{"s3-007-293847485-724784"},
|
||||
"X-Amz-Server-Side-Encryption-Context": []string{"{\"bucket\": \"some-bucket\"}"},
|
||||
}, ShouldFail: false}, // 4
|
||||
{Header: http.Header{
|
||||
"X-Amz-Server-Side-Encryption": []string{"aws:kms"},
|
||||
"X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": []string{"s3-007-293847485-724784"},
|
||||
"X-Amz-Server-Side-Encryption-Context": []string{"{\"bucket\": \"some-bucket\"}"},
|
||||
}, ShouldFail: false}, // 5
|
||||
{Header: http.Header{
|
||||
"X-Amz-Server-Side-Encryption": []string{"AES256"},
|
||||
"X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": []string{"s3-007-293847485-724784"},
|
||||
"X-Amz-Server-Side-Encryption-Context": []string{"{\"bucket\": \"some-bucket\"}"},
|
||||
}, ShouldFail: true}, // 6
|
||||
{Header: http.Header{
|
||||
"X-Amz-Server-Side-Encryption": []string{"aws:kms"},
|
||||
"X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": []string{"s3-007-293847485-724784"},
|
||||
"X-Amz-Server-Side-Encryption-Context": []string{"{\"bucket\": \"some-bucket\""}, // invalid JSON
|
||||
}, ShouldFail: true}, // 7
|
||||
|
||||
}
|
||||
|
||||
func TestKMSParseHTTP(t *testing.T) {
|
||||
for i, test := range kmsParseHTTPTests {
|
||||
_, _, err := S3KMS.ParseHTTP(test.Header)
|
||||
if err == nil && test.ShouldFail {
|
||||
t.Errorf("Test %d: should fail but succeeded", i)
|
||||
}
|
||||
if err != nil && !test.ShouldFail {
|
||||
t.Errorf("Test %d: should pass but failed with: %v", i, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var s3IsRequestedTests = []struct {
|
||||
Header http.Header
|
||||
Expected bool
|
||||
|
||||
+13
-1
@@ -39,7 +39,7 @@ func newBgHealSequence(numDisks int) *healSequence {
|
||||
hs := madmin.HealOpts{
|
||||
// Remove objects that do not have read-quorum
|
||||
Remove: true,
|
||||
ScanMode: madmin.HealDeepScan,
|
||||
ScanMode: madmin.HealNormalScan,
|
||||
}
|
||||
|
||||
return &healSequence{
|
||||
@@ -60,6 +60,18 @@ func newBgHealSequence(numDisks int) *healSequence {
|
||||
}
|
||||
}
|
||||
|
||||
func getLocalBackgroundHealStatus() madmin.BgHealState {
|
||||
backgroundSequence, ok := globalSweepHealState.getHealSequenceByToken(bgHealingUUID)
|
||||
if !ok {
|
||||
return madmin.BgHealState{}
|
||||
}
|
||||
|
||||
return madmin.BgHealState{
|
||||
ScannedItemsCount: backgroundSequence.scannedItemsCount,
|
||||
LastHealActivity: backgroundSequence.lastHealActivity,
|
||||
}
|
||||
}
|
||||
|
||||
func initDailyHeal() {
|
||||
go startDailyHeal()
|
||||
}
|
||||
|
||||
@@ -55,7 +55,7 @@ func sweepRound(ctx context.Context, objAPI ObjectLayer) error {
|
||||
zeroDynamicTimeout := newDynamicTimeout(zeroDuration, zeroDuration)
|
||||
|
||||
// General lock so we avoid parallel daily sweep by different instances.
|
||||
sweepLock := globalNSMutex.NewNSLock("system", "daily-sweep")
|
||||
sweepLock := globalNSMutex.NewNSLock(ctx, "system", "daily-sweep")
|
||||
if err := sweepLock.GetLock(zeroDynamicTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -303,7 +303,7 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
|
||||
data := r.Reader
|
||||
fs := cfs.FSObjects
|
||||
// Lock the object.
|
||||
objectLock := fs.nsMutex.NewNSLock(bucket, object)
|
||||
objectLock := fs.nsMutex.NewNSLock(ctx, bucket, object)
|
||||
if err := objectLock.GetLock(globalObjectTimeout); err != nil {
|
||||
return objInfo, err
|
||||
}
|
||||
@@ -489,7 +489,7 @@ func (cfs *cacheFSObjects) NewMultipartUpload(ctx context.Context, bucket, objec
|
||||
// moveBucketToTrash clears cacheFSObjects of bucket contents and moves it to trash folder.
|
||||
func (cfs *cacheFSObjects) moveBucketToTrash(ctx context.Context, bucket string) (err error) {
|
||||
fs := cfs.FSObjects
|
||||
bucketLock := fs.nsMutex.NewNSLock(bucket, "")
|
||||
bucketLock := fs.nsMutex.NewNSLock(ctx, bucket, "")
|
||||
if err = bucketLock.GetLock(globalObjectTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1238,6 +1238,17 @@ func putOpts(ctx context.Context, r *http.Request, bucket, object string, metada
|
||||
opts.UserDefined = metadata
|
||||
return
|
||||
}
|
||||
if crypto.S3KMS.IsRequested(r.Header) {
|
||||
keyID, context, err := crypto.S3KMS.ParseHTTP(r.Header)
|
||||
if err != nil {
|
||||
return ObjectOptions{}, err
|
||||
}
|
||||
sseKms, err := encrypt.NewSSEKMS(keyID, context)
|
||||
if err != nil {
|
||||
return ObjectOptions{}, err
|
||||
}
|
||||
return ObjectOptions{ServerSideEncryption: sseKms, UserDefined: metadata}, nil
|
||||
}
|
||||
// default case of passing encryption headers and UserDefined metadata to backend
|
||||
return getDefaultOpts(r.Header, false, metadata)
|
||||
}
|
||||
|
||||
+19
-7
@@ -20,6 +20,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
@@ -288,7 +289,7 @@ func (endpoints EndpointList) UpdateIsLocal() error {
|
||||
|
||||
// localEndpointsMemUsage - returns ServerMemUsageInfo for only the
|
||||
// local endpoints from given list of endpoints
|
||||
func localEndpointsMemUsage(endpoints EndpointList) ServerMemUsageInfo {
|
||||
func localEndpointsMemUsage(endpoints EndpointList, r *http.Request) ServerMemUsageInfo {
|
||||
var memUsages []mem.Usage
|
||||
var historicUsages []mem.Usage
|
||||
scratchSpace := map[string]bool{}
|
||||
@@ -303,8 +304,12 @@ func localEndpointsMemUsage(endpoints EndpointList) ServerMemUsageInfo {
|
||||
scratchSpace[endpoint.Host] = true
|
||||
}
|
||||
}
|
||||
addr := r.Host
|
||||
if globalIsDistXL {
|
||||
addr = GetLocalPeer(endpoints)
|
||||
}
|
||||
return ServerMemUsageInfo{
|
||||
Addr: GetLocalPeer(endpoints),
|
||||
Addr: addr,
|
||||
Usage: memUsages,
|
||||
HistoricUsage: historicUsages,
|
||||
}
|
||||
@@ -312,7 +317,7 @@ func localEndpointsMemUsage(endpoints EndpointList) ServerMemUsageInfo {
|
||||
|
||||
// localEndpointsCPULoad - returns ServerCPULoadInfo for only the
|
||||
// local endpoints from given list of endpoints
|
||||
func localEndpointsCPULoad(endpoints EndpointList) ServerCPULoadInfo {
|
||||
func localEndpointsCPULoad(endpoints EndpointList, r *http.Request) ServerCPULoadInfo {
|
||||
var cpuLoads []cpu.Load
|
||||
var historicLoads []cpu.Load
|
||||
scratchSpace := map[string]bool{}
|
||||
@@ -327,8 +332,12 @@ func localEndpointsCPULoad(endpoints EndpointList) ServerCPULoadInfo {
|
||||
scratchSpace[endpoint.Host] = true
|
||||
}
|
||||
}
|
||||
addr := r.Host
|
||||
if globalIsDistXL {
|
||||
addr = GetLocalPeer(endpoints)
|
||||
}
|
||||
return ServerCPULoadInfo{
|
||||
Addr: GetLocalPeer(endpoints),
|
||||
Addr: addr,
|
||||
Load: cpuLoads,
|
||||
HistoricLoad: historicLoads,
|
||||
}
|
||||
@@ -336,7 +345,7 @@ func localEndpointsCPULoad(endpoints EndpointList) ServerCPULoadInfo {
|
||||
|
||||
// localEndpointsDrivePerf - returns ServerDrivesPerfInfo for only the
|
||||
// local endpoints from given list of endpoints
|
||||
func localEndpointsDrivePerf(endpoints EndpointList) ServerDrivesPerfInfo {
|
||||
func localEndpointsDrivePerf(endpoints EndpointList, r *http.Request) ServerDrivesPerfInfo {
|
||||
var dps []disk.Performance
|
||||
for _, endpoint := range endpoints {
|
||||
// Only proceed for local endpoints
|
||||
@@ -351,9 +360,12 @@ func localEndpointsDrivePerf(endpoints EndpointList) ServerDrivesPerfInfo {
|
||||
dps = append(dps, dp)
|
||||
}
|
||||
}
|
||||
|
||||
addr := r.Host
|
||||
if globalIsDistXL {
|
||||
addr = GetLocalPeer(endpoints)
|
||||
}
|
||||
return ServerDrivesPerfInfo{
|
||||
Addr: GetLocalPeer(endpoints),
|
||||
Addr: addr,
|
||||
Perf: dps,
|
||||
}
|
||||
}
|
||||
|
||||
+1
-4
@@ -77,10 +77,7 @@ func (e *Erasure) DecodeDataBlocks(data [][]byte) error {
|
||||
if !needsReconstruction {
|
||||
return nil
|
||||
}
|
||||
if err := e.encoder.ReconstructData(data); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
return e.encoder.ReconstructData(data)
|
||||
}
|
||||
|
||||
// DecodeDataAndParityBlocks decodes the given erasure-coded data and verifies it.
|
||||
|
||||
+1
-1
@@ -491,7 +491,7 @@ func formatXLGetDeploymentID(refFormat *formatXLV3, formats []*formatXLV3) (stri
|
||||
func formatXLFixDeploymentID(ctx context.Context, endpoints EndpointList, storageDisks []StorageAPI, refFormat *formatXLV3) (err error) {
|
||||
// Acquire lock on format.json
|
||||
mutex := newNSLock(globalIsDistXL)
|
||||
formatLock := mutex.NewNSLock(minioMetaBucket, formatConfigFile)
|
||||
formatLock := mutex.NewNSLock(ctx, minioMetaBucket, formatConfigFile)
|
||||
if err = formatLock.GetLock(globalHealingTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -634,7 +634,7 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
|
||||
}
|
||||
|
||||
// Hold write lock on the object.
|
||||
destLock := fs.nsMutex.NewNSLock(bucket, object)
|
||||
destLock := fs.nsMutex.NewNSLock(ctx, bucket, object)
|
||||
if err = destLock.GetLock(globalObjectTimeout); err != nil {
|
||||
return oi, err
|
||||
}
|
||||
|
||||
+10
-10
@@ -286,7 +286,7 @@ func (fs *FSObjects) statBucketDir(ctx context.Context, bucket string) (os.FileI
|
||||
// MakeBucketWithLocation - create a new bucket, returns if it
|
||||
// already exists.
|
||||
func (fs *FSObjects) MakeBucketWithLocation(ctx context.Context, bucket, location string) error {
|
||||
bucketLock := fs.nsMutex.NewNSLock(bucket, "")
|
||||
bucketLock := fs.nsMutex.NewNSLock(ctx, bucket, "")
|
||||
if err := bucketLock.GetLock(globalObjectTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -309,7 +309,7 @@ func (fs *FSObjects) MakeBucketWithLocation(ctx context.Context, bucket, locatio
|
||||
|
||||
// GetBucketInfo - fetch bucket metadata info.
|
||||
func (fs *FSObjects) GetBucketInfo(ctx context.Context, bucket string) (bi BucketInfo, e error) {
|
||||
bucketLock := fs.nsMutex.NewNSLock(bucket, "")
|
||||
bucketLock := fs.nsMutex.NewNSLock(ctx, bucket, "")
|
||||
if e := bucketLock.GetRLock(globalObjectTimeout); e != nil {
|
||||
return bi, e
|
||||
}
|
||||
@@ -372,7 +372,7 @@ func (fs *FSObjects) ListBuckets(ctx context.Context) ([]BucketInfo, error) {
|
||||
// DeleteBucket - delete a bucket and all the metadata associated
|
||||
// with the bucket including pending multipart, object metadata.
|
||||
func (fs *FSObjects) DeleteBucket(ctx context.Context, bucket string) error {
|
||||
bucketLock := fs.nsMutex.NewNSLock(bucket, "")
|
||||
bucketLock := fs.nsMutex.NewNSLock(ctx, bucket, "")
|
||||
if err := bucketLock.GetLock(globalObjectTimeout); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return err
|
||||
@@ -408,7 +408,7 @@ func (fs *FSObjects) DeleteBucket(ctx context.Context, bucket string) error {
|
||||
func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (oi ObjectInfo, e error) {
|
||||
cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
|
||||
if !cpSrcDstSame {
|
||||
objectDWLock := fs.nsMutex.NewNSLock(dstBucket, dstObject)
|
||||
objectDWLock := fs.nsMutex.NewNSLock(ctx, dstBucket, dstObject)
|
||||
if err := objectDWLock.GetLock(globalObjectTimeout); err != nil {
|
||||
return oi, err
|
||||
}
|
||||
@@ -480,7 +480,7 @@ func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
|
||||
|
||||
if lockType != noLock {
|
||||
// Lock the object before reading.
|
||||
lock := fs.nsMutex.NewNSLock(bucket, object)
|
||||
lock := fs.nsMutex.NewNSLock(ctx, bucket, object)
|
||||
switch lockType {
|
||||
case writeLock:
|
||||
if err = lock.GetLock(globalObjectTimeout); err != nil {
|
||||
@@ -567,7 +567,7 @@ func (fs *FSObjects) GetObject(ctx context.Context, bucket, object string, offse
|
||||
}
|
||||
|
||||
// Lock the object before reading.
|
||||
objectLock := fs.nsMutex.NewNSLock(bucket, object)
|
||||
objectLock := fs.nsMutex.NewNSLock(ctx, bucket, object)
|
||||
if err := objectLock.GetRLock(globalObjectTimeout); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return err
|
||||
@@ -739,7 +739,7 @@ func (fs *FSObjects) getObjectInfo(ctx context.Context, bucket, object string) (
|
||||
// getObjectInfoWithLock - reads object metadata and replies back ObjectInfo.
|
||||
func (fs *FSObjects) getObjectInfoWithLock(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) {
|
||||
// Lock the object before reading.
|
||||
objectLock := fs.nsMutex.NewNSLock(bucket, object)
|
||||
objectLock := fs.nsMutex.NewNSLock(ctx, bucket, object)
|
||||
if err := objectLock.GetRLock(globalObjectTimeout); err != nil {
|
||||
return oi, err
|
||||
}
|
||||
@@ -764,7 +764,7 @@ func (fs *FSObjects) getObjectInfoWithLock(ctx context.Context, bucket, object s
|
||||
func (fs *FSObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (oi ObjectInfo, e error) {
|
||||
oi, err := fs.getObjectInfoWithLock(ctx, bucket, object)
|
||||
if err == errCorruptedFormat || err == io.EOF {
|
||||
objectLock := fs.nsMutex.NewNSLock(bucket, object)
|
||||
objectLock := fs.nsMutex.NewNSLock(ctx, bucket, object)
|
||||
if err = objectLock.GetLock(globalObjectTimeout); err != nil {
|
||||
return oi, toObjectErr(err, bucket, object)
|
||||
}
|
||||
@@ -810,7 +810,7 @@ func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
// Lock the object.
|
||||
objectLock := fs.nsMutex.NewNSLock(bucket, object)
|
||||
objectLock := fs.nsMutex.NewNSLock(ctx, bucket, object)
|
||||
if err := objectLock.GetLock(globalObjectTimeout); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return objInfo, err
|
||||
@@ -965,7 +965,7 @@ func (fs *FSObjects) DeleteObjects(ctx context.Context, bucket string, objects [
|
||||
// and there are no rollbacks supported.
|
||||
func (fs *FSObjects) DeleteObject(ctx context.Context, bucket, object string) error {
|
||||
// Acquire a write lock before deleting the object.
|
||||
objectLock := fs.nsMutex.NewNSLock(bucket, object)
|
||||
objectLock := fs.nsMutex.NewNSLock(ctx, bucket, object)
|
||||
if err := objectLock.GetLock(globalOperationTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
|
||||
@@ -166,7 +167,7 @@ func FromMinioClientListMultipartsInfo(lmur minio.ListMultipartUploadsResult) Li
|
||||
// FromMinioClientObjectInfo converts minio ObjectInfo to gateway ObjectInfo
|
||||
func FromMinioClientObjectInfo(bucket string, oi minio.ObjectInfo) ObjectInfo {
|
||||
userDefined := FromMinioClientMetadata(oi.Metadata)
|
||||
userDefined["Content-Type"] = oi.ContentType
|
||||
userDefined[xhttp.ContentType] = oi.ContentType
|
||||
|
||||
return ObjectInfo{
|
||||
Bucket: bucket,
|
||||
@@ -176,7 +177,7 @@ func FromMinioClientObjectInfo(bucket string, oi minio.ObjectInfo) ObjectInfo {
|
||||
ETag: canonicalizeETag(oi.ETag),
|
||||
UserDefined: userDefined,
|
||||
ContentType: oi.ContentType,
|
||||
ContentEncoding: oi.Metadata.Get("Content-Encoding"),
|
||||
ContentEncoding: oi.Metadata.Get(xhttp.ContentEncoding),
|
||||
StorageClass: oi.StorageClass,
|
||||
Expires: oi.Expires,
|
||||
}
|
||||
|
||||
+5
-6
@@ -48,7 +48,7 @@ var (
|
||||
|
||||
// RegisterGatewayCommand registers a new command for gateway.
|
||||
func RegisterGatewayCommand(cmd cli.Command) error {
|
||||
cmd.Flags = append(append(cmd.Flags, append(cmd.Flags, ServerFlags...)...), GlobalFlags...)
|
||||
cmd.Flags = append(append(cmd.Flags, ServerFlags...), GlobalFlags...)
|
||||
gatewayCmd.Subcommands = append(gatewayCmd.Subcommands, cmd)
|
||||
return nil
|
||||
}
|
||||
@@ -121,7 +121,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
// to IPv6 address ie minio will start listening on IPv6 address whereas another
|
||||
// (non-)minio process is listening on IPv4 of given port.
|
||||
// To avoid this error situation we check for port availability.
|
||||
logger.FatalIf(checkPortAvailability(globalMinioPort), "Unable to start the gateway")
|
||||
logger.FatalIf(checkPortAvailability(globalMinioHost, globalMinioPort), "Unable to start the gateway")
|
||||
|
||||
// Check and load TLS certificates.
|
||||
var err error
|
||||
@@ -158,9 +158,6 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
registerSTSRouter(router)
|
||||
}
|
||||
|
||||
// initialize globalTrace system
|
||||
globalTrace = NewTraceSys(context.Background(), globalEndpoints)
|
||||
|
||||
enableConfigOps := globalEtcdClient != nil && gatewayName == "nas"
|
||||
enableIAMOps := globalEtcdClient != nil
|
||||
|
||||
@@ -181,9 +178,10 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
|
||||
// Currently only NAS and S3 gateway support encryption headers.
|
||||
encryptionEnabled := gatewayName == "s3" || gatewayName == "nas"
|
||||
allowSSEKMS := gatewayName == "s3" // Only S3 can support SSE-KMS (as pass-through)
|
||||
|
||||
// Add API router.
|
||||
registerAPIRouter(router, encryptionEnabled)
|
||||
registerAPIRouter(router, encryptionEnabled, allowSSEKMS)
|
||||
|
||||
var getCert certs.GetCertificateFunc
|
||||
if globalTLSCerts != nil {
|
||||
@@ -245,6 +243,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
|
||||
// This is only to uniquely identify each gateway deployments.
|
||||
globalDeploymentID = os.Getenv("MINIO_GATEWAY_DEPLOYMENT_ID")
|
||||
logger.SetDeploymentID(globalDeploymentID)
|
||||
|
||||
var cacheConfig = globalServerConfig.GetCacheConfig()
|
||||
if len(cacheConfig.Drives) > 0 {
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -34,6 +35,36 @@ func TestRegisterGatewayCommand(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Test running a registered gateway command with a flag
|
||||
func TestRunRegisteredGatewayCommand(t *testing.T) {
|
||||
var err error
|
||||
|
||||
flagName := "test-flag"
|
||||
flagValue := "foo"
|
||||
|
||||
cmd := cli.Command{
|
||||
Name: "test-run-with-flag",
|
||||
Flags: []cli.Flag{
|
||||
cli.StringFlag{Name: flagName},
|
||||
},
|
||||
Action: func(ctx *cli.Context) {
|
||||
if actual := ctx.String(flagName); actual != flagValue {
|
||||
t.Errorf("value of %s expects %s, but got %s", flagName, flagValue, actual)
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
err = RegisterGatewayCommand(cmd)
|
||||
if err != nil {
|
||||
t.Errorf("RegisterGatewayCommand got unexpected error: %s", err)
|
||||
}
|
||||
|
||||
if err = newApp("minio").Run(
|
||||
[]string{"minio", "gateway", cmd.Name, fmt.Sprintf("--%s", flagName), flagValue}); err != nil {
|
||||
t.Errorf("running registered gateway command got unexpected error: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Test parseGatewayEndpoint
|
||||
func TestParseGatewayEndpoint(t *testing.T) {
|
||||
testCases := []struct {
|
||||
|
||||
@@ -29,7 +29,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
b2 "github.com/minio/blazer/base"
|
||||
b2 "github.com/kurin/blazer/base"
|
||||
"github.com/minio/cli"
|
||||
miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
@@ -147,6 +147,29 @@ func b2ToObjectError(err error, params ...string) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
code, msgCode, msg := b2.MsgCode(err)
|
||||
if code == 0 {
|
||||
// We don't interpret non B2 errors. B2 errors have statusCode
|
||||
// to help us convert them to S3 object errors.
|
||||
return err
|
||||
}
|
||||
|
||||
objErr := b2MsgCodeToObjectError(code, msgCode, msg, params...)
|
||||
if objErr == nil {
|
||||
return err
|
||||
}
|
||||
return objErr
|
||||
}
|
||||
|
||||
func b2MsgCodeToObjectError(code int, msgCode string, msg string, params ...string) error {
|
||||
// Following code is a non-exhaustive check to convert
|
||||
// B2 errors into S3 compatible errors.
|
||||
//
|
||||
// For a more complete information - https://www.backblaze.com/b2/docs/
|
||||
|
||||
var err error
|
||||
|
||||
bucket := ""
|
||||
object := ""
|
||||
uploadID := ""
|
||||
@@ -160,18 +183,7 @@ func b2ToObjectError(err error, params ...string) error {
|
||||
uploadID = params[2]
|
||||
}
|
||||
|
||||
// Following code is a non-exhaustive check to convert
|
||||
// B2 errors into S3 compatible errors.
|
||||
//
|
||||
// For a more complete information - https://www.backblaze.com/b2/docs/
|
||||
statusCode, code, msg := b2.Code(err)
|
||||
if statusCode == 0 {
|
||||
// We don't interpret non B2 errors. B2 errors have statusCode
|
||||
// to help us convert them to S3 object errors.
|
||||
return err
|
||||
}
|
||||
|
||||
switch code {
|
||||
switch msgCode {
|
||||
case "duplicate_bucket_name":
|
||||
err = minio.BucketAlreadyOwnedByYou{Bucket: bucket}
|
||||
case "bad_request":
|
||||
@@ -343,7 +355,7 @@ func (l *b2Objects) ListObjects(ctx context.Context, bucket string, prefix strin
|
||||
Name: file.Name,
|
||||
ModTime: file.Timestamp,
|
||||
Size: file.Size,
|
||||
ETag: minio.ToS3ETag(file.Info.ID),
|
||||
ETag: minio.ToS3ETag(file.ID),
|
||||
ContentType: file.Info.ContentType,
|
||||
UserDefined: file.Info.Info,
|
||||
})
|
||||
@@ -386,7 +398,7 @@ func (l *b2Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuat
|
||||
Name: file.Name,
|
||||
ModTime: file.Timestamp,
|
||||
Size: file.Size,
|
||||
ETag: minio.ToS3ETag(file.Info.ID),
|
||||
ETag: minio.ToS3ETag(file.ID),
|
||||
ContentType: file.Info.ContentType,
|
||||
UserDefined: file.Info.Info,
|
||||
})
|
||||
@@ -462,7 +474,7 @@ func (l *b2Objects) GetObjectInfo(ctx context.Context, bucket string, object str
|
||||
return minio.ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: object,
|
||||
ETag: minio.ToS3ETag(fi.ID),
|
||||
ETag: minio.ToS3ETag(f.ID),
|
||||
Size: fi.Size,
|
||||
ModTime: fi.Timestamp,
|
||||
ContentType: fi.ContentType,
|
||||
@@ -569,7 +581,7 @@ func (l *b2Objects) PutObject(ctx context.Context, bucket string, object string,
|
||||
return minio.ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: object,
|
||||
ETag: minio.ToS3ETag(fi.ID),
|
||||
ETag: minio.ToS3ETag(f.ID),
|
||||
Size: fi.Size,
|
||||
ModTime: fi.Timestamp,
|
||||
ContentType: fi.ContentType,
|
||||
@@ -617,7 +629,7 @@ func (l *b2Objects) ListMultipartUploads(ctx context.Context, bucket string, pre
|
||||
if maxUploads > 100 {
|
||||
maxUploads = 100
|
||||
}
|
||||
largeFiles, nextMarker, err := bkt.ListUnfinishedLargeFiles(l.ctx, uploadIDMarker, maxUploads)
|
||||
largeFiles, nextMarker, err := bkt.ListUnfinishedLargeFiles(l.ctx, maxUploads, uploadIDMarker)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return lmi, b2ToObjectError(err, bucket)
|
||||
@@ -677,7 +689,7 @@ func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object str
|
||||
}
|
||||
|
||||
hr := newB2Reader(data, data.Size())
|
||||
sha1, err := fc.UploadPart(l.ctx, hr, sha1AtEOF, int(hr.Size()), partID)
|
||||
_, err = fc.UploadPart(l.ctx, hr, sha1AtEOF, int(hr.Size()), partID)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return pi, b2ToObjectError(err, bucket, object, uploadID)
|
||||
@@ -686,7 +698,7 @@ func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object str
|
||||
return minio.PartInfo{
|
||||
PartNumber: partID,
|
||||
LastModified: minio.UTCNow(),
|
||||
ETag: minio.ToS3ETag(sha1),
|
||||
ETag: minio.ToS3ETag(fmt.Sprintf("%x", hr.sha1Hash.Sum(nil))),
|
||||
Size: data.Size(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -20,8 +20,6 @@ import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
b2 "github.com/minio/blazer/base"
|
||||
|
||||
minio "github.com/minio/minio/cmd"
|
||||
)
|
||||
|
||||
@@ -41,71 +39,6 @@ func TestB2ObjectError(t *testing.T) {
|
||||
{
|
||||
[]string{}, fmt.Errorf("Non B2 Error"), fmt.Errorf("Non B2 Error"),
|
||||
},
|
||||
{
|
||||
[]string{"bucket"}, b2.Error{
|
||||
StatusCode: 1,
|
||||
Code: "duplicate_bucket_name",
|
||||
}, minio.BucketAlreadyOwnedByYou{
|
||||
Bucket: "bucket",
|
||||
},
|
||||
},
|
||||
{
|
||||
[]string{"bucket"}, b2.Error{
|
||||
StatusCode: 1,
|
||||
Code: "bad_request",
|
||||
}, minio.BucketNotFound{
|
||||
Bucket: "bucket",
|
||||
},
|
||||
},
|
||||
{
|
||||
[]string{"bucket", "object"}, b2.Error{
|
||||
StatusCode: 1,
|
||||
Code: "bad_request",
|
||||
}, minio.ObjectNameInvalid{
|
||||
Bucket: "bucket",
|
||||
Object: "object",
|
||||
},
|
||||
},
|
||||
{
|
||||
[]string{"bucket"}, b2.Error{
|
||||
StatusCode: 1,
|
||||
Code: "bad_bucket_id",
|
||||
}, minio.BucketNotFound{Bucket: "bucket"},
|
||||
},
|
||||
{
|
||||
[]string{"bucket", "object"}, b2.Error{
|
||||
StatusCode: 1,
|
||||
Code: "file_not_present",
|
||||
}, minio.ObjectNotFound{
|
||||
Bucket: "bucket",
|
||||
Object: "object",
|
||||
},
|
||||
},
|
||||
{
|
||||
[]string{"bucket", "object"}, b2.Error{
|
||||
StatusCode: 1,
|
||||
Code: "not_found",
|
||||
}, minio.ObjectNotFound{
|
||||
Bucket: "bucket",
|
||||
Object: "object",
|
||||
},
|
||||
},
|
||||
{
|
||||
[]string{"bucket"}, b2.Error{
|
||||
StatusCode: 1,
|
||||
Code: "cannot_delete_non_empty_bucket",
|
||||
}, minio.BucketNotEmpty{
|
||||
Bucket: "bucket",
|
||||
},
|
||||
},
|
||||
{
|
||||
[]string{"bucket", "object", "uploadID"}, b2.Error{
|
||||
StatusCode: 1,
|
||||
Message: "No active upload for",
|
||||
}, minio.InvalidUploadID{
|
||||
UploadID: "uploadID",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
@@ -117,3 +50,97 @@ func TestB2ObjectError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test b2 msg code to object error.
|
||||
func TestB2MsgCodeToObjectError(t *testing.T) {
|
||||
testCases := []struct {
|
||||
params []string
|
||||
code int
|
||||
msgCode string
|
||||
msg string
|
||||
expectedErr error
|
||||
}{
|
||||
{
|
||||
[]string{"bucket"},
|
||||
1,
|
||||
"duplicate_bucket_name",
|
||||
"",
|
||||
minio.BucketAlreadyOwnedByYou{
|
||||
Bucket: "bucket",
|
||||
},
|
||||
},
|
||||
{
|
||||
[]string{"bucket"},
|
||||
1,
|
||||
"bad_request",
|
||||
"",
|
||||
minio.BucketNotFound{
|
||||
Bucket: "bucket",
|
||||
},
|
||||
},
|
||||
{
|
||||
[]string{"bucket", "object"},
|
||||
1,
|
||||
"bad_request",
|
||||
"",
|
||||
minio.ObjectNameInvalid{
|
||||
Bucket: "bucket",
|
||||
Object: "object",
|
||||
},
|
||||
},
|
||||
{
|
||||
[]string{"bucket"},
|
||||
1,
|
||||
"bad_bucket_id",
|
||||
"",
|
||||
minio.BucketNotFound{Bucket: "bucket"},
|
||||
},
|
||||
{
|
||||
[]string{"bucket", "object"},
|
||||
1,
|
||||
"file_not_present",
|
||||
"",
|
||||
minio.ObjectNotFound{
|
||||
Bucket: "bucket",
|
||||
Object: "object",
|
||||
},
|
||||
},
|
||||
{
|
||||
[]string{"bucket", "object"},
|
||||
1,
|
||||
"not_found",
|
||||
"",
|
||||
minio.ObjectNotFound{
|
||||
Bucket: "bucket",
|
||||
Object: "object",
|
||||
},
|
||||
},
|
||||
{
|
||||
[]string{"bucket"},
|
||||
1,
|
||||
"cannot_delete_non_empty_bucket",
|
||||
"",
|
||||
minio.BucketNotEmpty{
|
||||
Bucket: "bucket",
|
||||
},
|
||||
},
|
||||
{
|
||||
[]string{"bucket", "object", "uploadID"},
|
||||
1,
|
||||
"",
|
||||
"No active upload for",
|
||||
minio.InvalidUploadID{
|
||||
UploadID: "uploadID",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
actualErr := b2MsgCodeToObjectError(testCase.code, testCase.msgCode, testCase.msg, testCase.params...)
|
||||
if actualErr != nil {
|
||||
if actualErr.Error() != testCase.expectedErr.Error() {
|
||||
t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.expectedErr, actualErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -442,7 +442,8 @@ func (l *s3EncObjects) PutObject(ctx context.Context, bucket string, object stri
|
||||
// Decide if sse options needed to be passed to backend
|
||||
if opts.ServerSideEncryption != nil &&
|
||||
((minio.GlobalGatewaySSE.SSEC() && opts.ServerSideEncryption.Type() == encrypt.SSEC) ||
|
||||
(minio.GlobalGatewaySSE.SSES3() && opts.ServerSideEncryption.Type() == encrypt.S3)) {
|
||||
(minio.GlobalGatewaySSE.SSES3() && opts.ServerSideEncryption.Type() == encrypt.S3) ||
|
||||
opts.ServerSideEncryption.Type() == encrypt.KMS) {
|
||||
sseOpts = opts.ServerSideEncryption
|
||||
}
|
||||
if opts.ServerSideEncryption == nil {
|
||||
|
||||
+19
-21
@@ -28,6 +28,7 @@ import (
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/dns"
|
||||
"github.com/minio/minio/pkg/handlers"
|
||||
@@ -261,10 +262,10 @@ func (h cacheControlHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if hasSuffix(r.URL.Path, ".js") || r.URL.Path == minioReservedBucketPath+"/favicon.ico" {
|
||||
// For assets set cache expiry of one year. For each release, the name
|
||||
// of the asset name will change and hence it can not be served from cache.
|
||||
w.Header().Set("Cache-Control", "max-age=31536000")
|
||||
w.Header().Set(xhttp.CacheControl, "max-age=31536000")
|
||||
} else {
|
||||
// For non asset requests we serve index.html which will never be cached.
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
w.Header().Set(xhttp.CacheControl, "no-store")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -294,7 +295,7 @@ func (h minioReservedBucketHandler) ServeHTTP(w http.ResponseWriter, r *http.Req
|
||||
default:
|
||||
// For all other requests reject access to reserved
|
||||
// buckets
|
||||
bucketName, _ := urlPath2BucketObjectName(r.URL.Path)
|
||||
bucketName, _ := request2BucketObjectName(r)
|
||||
if isMinioReservedBucket(bucketName) || isMinioMetaBucket(bucketName) {
|
||||
writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrAllAccessDisabled), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
@@ -380,15 +381,15 @@ type resourceHandler struct {
|
||||
// setCorsHandler handler for CORS (Cross Origin Resource Sharing)
|
||||
func setCorsHandler(h http.Handler) http.Handler {
|
||||
commonS3Headers := []string{
|
||||
"Date",
|
||||
"ETag",
|
||||
"Server",
|
||||
"Connection",
|
||||
"Accept-Ranges",
|
||||
"Content-Range",
|
||||
"Content-Encoding",
|
||||
"Content-Length",
|
||||
"Content-Type",
|
||||
xhttp.Date,
|
||||
xhttp.ETag,
|
||||
xhttp.ServerInfo,
|
||||
xhttp.Connection,
|
||||
xhttp.AcceptRanges,
|
||||
xhttp.ContentRange,
|
||||
xhttp.ContentEncoding,
|
||||
xhttp.ContentLength,
|
||||
xhttp.ContentType,
|
||||
"X-Amz*",
|
||||
"x-amz*",
|
||||
"*",
|
||||
@@ -494,7 +495,7 @@ var notimplementedObjectResourceNames = map[string]bool{
|
||||
|
||||
// Resource handler ServeHTTP() wrapper
|
||||
func (h resourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
bucketName, objectName := urlPath2BucketObjectName(r.URL.Path)
|
||||
bucketName, objectName := request2BucketObjectName(r)
|
||||
|
||||
// If bucketName is present and not objectName check for bucket level resource queries.
|
||||
if bucketName != "" && objectName == "" {
|
||||
@@ -696,7 +697,8 @@ func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
|
||||
return
|
||||
}
|
||||
|
||||
bucket, object := urlPath2BucketObjectName(r.URL.Path)
|
||||
bucket, object := request2BucketObjectName(r)
|
||||
|
||||
// ListBucket requests should be handled at current endpoint as
|
||||
// all buckets data can be fetched from here.
|
||||
if r.Method == http.MethodGet && bucket == "" && object == "" {
|
||||
@@ -749,7 +751,7 @@ func setBucketForwardingHandler(h http.Handler) http.Handler {
|
||||
return bucketForwardingHandler{fwd, h}
|
||||
}
|
||||
|
||||
// customHeaderHandler sets x-amz-request-id, x-minio-deployment-id header.
|
||||
// customHeaderHandler sets x-amz-request-id header.
|
||||
// Previously, this value was set right before a response was sent to
|
||||
// the client. So, logger and Error response XML were not using this
|
||||
// value. This is set here so that this header can be logged as
|
||||
@@ -763,12 +765,8 @@ func addCustomHeaders(h http.Handler) http.Handler {
|
||||
}
|
||||
|
||||
func (s customHeaderHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// Set custom headers such as x-amz-request-id and x-minio-deployment-id
|
||||
// for each request.
|
||||
w.Header().Set(responseRequestIDKey, mustGetRequestID(UTCNow()))
|
||||
if globalDeploymentID != "" {
|
||||
w.Header().Set(responseDeploymentIDKey, globalDeploymentID)
|
||||
}
|
||||
// Set custom headers such as x-amz-request-id for each request.
|
||||
w.Header().Set(xhttp.AmzRequestID, mustGetRequestID(UTCNow()))
|
||||
s.handler.ServeHTTP(logger.NewResponseWriter(w), r)
|
||||
}
|
||||
|
||||
|
||||
+2
-1
@@ -35,6 +35,7 @@ import (
|
||||
"github.com/minio/minio/pkg/dns"
|
||||
iampolicy "github.com/minio/minio/pkg/iam/policy"
|
||||
"github.com/minio/minio/pkg/iam/validator"
|
||||
"github.com/minio/minio/pkg/pubsub"
|
||||
)
|
||||
|
||||
// minio configuration related constants.
|
||||
@@ -161,7 +162,7 @@ var (
|
||||
|
||||
// global Trace system to send HTTP request/response logs to
|
||||
// registered listeners
|
||||
globalTrace *HTTPTraceSys
|
||||
globalHTTPTrace = pubsub.New()
|
||||
|
||||
globalEndpoints EndpointList
|
||||
|
||||
|
||||
+22
-6
@@ -27,9 +27,11 @@ import (
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/handlers"
|
||||
xnet "github.com/minio/minio/pkg/net"
|
||||
)
|
||||
|
||||
// Parses location constraint from the incoming reader.
|
||||
@@ -219,8 +221,8 @@ func extractReqParams(r *http.Request) map[string]string {
|
||||
func extractRespElements(w http.ResponseWriter) map[string]string {
|
||||
|
||||
return map[string]string{
|
||||
"requestId": w.Header().Get(responseRequestIDKey),
|
||||
"content-length": w.Header().Get("Content-Length"),
|
||||
"requestId": w.Header().Get(xhttp.AmzRequestID),
|
||||
"content-length": w.Header().Get(xhttp.ContentLength),
|
||||
// Add more fields here.
|
||||
}
|
||||
}
|
||||
@@ -326,24 +328,24 @@ func extractPostPolicyFormValues(ctx context.Context, form *multipart.Form) (fil
|
||||
// Log headers and body.
|
||||
func httpTraceAll(f http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if !globalTrace.HasTraceListeners() {
|
||||
if !globalHTTPTrace.HasSubscribers() {
|
||||
f.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
trace := Trace(f, true, w, r)
|
||||
globalTrace.Publish(trace)
|
||||
globalHTTPTrace.Publish(trace)
|
||||
}
|
||||
}
|
||||
|
||||
// Log only the headers.
|
||||
func httpTraceHdrs(f http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if !globalTrace.HasTraceListeners() {
|
||||
if !globalHTTPTrace.HasSubscribers() {
|
||||
f.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
trace := Trace(f, false, w, r)
|
||||
globalTrace.Publish(trace)
|
||||
globalHTTPTrace.Publish(trace)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -383,3 +385,17 @@ func notFoundHandlerJSON(w http.ResponseWriter, r *http.Request) {
|
||||
func notFoundHandler(w http.ResponseWriter, r *http.Request) {
|
||||
writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
|
||||
}
|
||||
|
||||
// gets host name for current node
|
||||
func getHostName(r *http.Request) (hostName string, err error) {
|
||||
var thisAddr *xnet.Host
|
||||
hostName = r.Host
|
||||
if globalIsDistXL {
|
||||
thisAddr, err = xnet.ParseHost(GetLocalPeer(globalEndpoints))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
hostName = thisAddr.String()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -62,6 +62,7 @@ func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if s.Backend.Type == Unknown {
|
||||
// ListBuckets to confirm gateway backend is up
|
||||
if _, err := objLayer.ListBuckets(ctx); err != nil {
|
||||
logger.LogOnceIf(ctx, err, struct{}{})
|
||||
writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
|
||||
return
|
||||
}
|
||||
|
||||
+27
-9
@@ -21,6 +21,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"runtime"
|
||||
@@ -28,7 +29,6 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
xnet "github.com/minio/minio/pkg/net"
|
||||
trace "github.com/minio/minio/pkg/trace"
|
||||
)
|
||||
|
||||
@@ -118,12 +118,26 @@ func (r *recordResponseWriter) Body() []byte {
|
||||
return r.body.Bytes()
|
||||
}
|
||||
|
||||
// getOpName sanitizes the operation name for mc
|
||||
func getOpName(name string) (op string) {
|
||||
op = strings.TrimPrefix(name, "github.com/minio/minio/cmd.")
|
||||
op = strings.TrimSuffix(op, "Handler-fm")
|
||||
op = strings.Replace(op, "objectAPIHandlers", "s3", 1)
|
||||
op = strings.Replace(op, "webAPIHandlers", "s3", 1)
|
||||
op = strings.Replace(op, "adminAPIHandlers", "admin", 1)
|
||||
op = strings.Replace(op, "(*storageRESTServer)", "internal", 1)
|
||||
op = strings.Replace(op, "(*peerRESTServer)", "internal", 1)
|
||||
op = strings.Replace(op, "(*lockRESTServer)", "internal", 1)
|
||||
op = strings.Replace(op, "stsAPIHandlers", "sts", 1)
|
||||
op = strings.Replace(op, "LivenessCheckHandler", "healthcheck", 1)
|
||||
op = strings.Replace(op, "ReadinessCheckHandler", "healthcheck", 1)
|
||||
return op
|
||||
}
|
||||
|
||||
// Trace gets trace of http request
|
||||
func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Request) trace.Info {
|
||||
|
||||
name := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
|
||||
name = strings.TrimPrefix(name, "github.com/minio/minio/cmd.")
|
||||
name = strings.TrimSuffix(name, "Handler-fm")
|
||||
name := getOpName(runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name())
|
||||
|
||||
bodyPlaceHolder := []byte("<BODY>")
|
||||
var reqBodyRecorder *recordRequest
|
||||
@@ -131,12 +145,16 @@ func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Requ
|
||||
t := trace.Info{FuncName: name}
|
||||
reqBodyRecorder = &recordRequest{Reader: r.Body, logBody: logBody}
|
||||
r.Body = ioutil.NopCloser(reqBodyRecorder)
|
||||
|
||||
host, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
|
||||
if err == nil {
|
||||
t.NodeName = host.Name
|
||||
t.NodeName = r.Host
|
||||
if globalIsDistXL {
|
||||
t.NodeName = GetLocalPeer(globalEndpoints)
|
||||
}
|
||||
rq := trace.RequestInfo{Time: time.Now().UTC(), Method: r.Method, Path: r.URL.Path, RawQuery: r.URL.RawQuery}
|
||||
// strip port from the host address
|
||||
if host, _, err := net.SplitHostPort(t.NodeName); err == nil {
|
||||
t.NodeName = host
|
||||
}
|
||||
|
||||
rq := trace.RequestInfo{Time: time.Now().UTC(), Method: r.Method, Path: r.URL.Path, RawQuery: r.URL.RawQuery, Client: r.RemoteAddr}
|
||||
rq.Headers = cloneHeader(r.Header)
|
||||
rq.Headers.Set("Content-Length", strconv.Itoa(int(r.ContentLength)))
|
||||
rq.Headers.Set("Host", r.Host)
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
/*
|
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package http
|
||||
|
||||
// Standard S3 HTTP response constants
|
||||
const (
|
||||
LastModified = "Last-Modified"
|
||||
Date = "Date"
|
||||
ETag = "ETag"
|
||||
ContentType = "Content-Type"
|
||||
ContentMD5 = "Content-Md5"
|
||||
ContentEncoding = "Content-Encoding"
|
||||
Expires = "Expires"
|
||||
ContentLength = "Content-Length"
|
||||
ContentLanguage = "Content-Language"
|
||||
ContentRange = "Content-Range"
|
||||
Connection = "Connection"
|
||||
AcceptRanges = "Accept-Ranges"
|
||||
AmzBucketRegion = "X-Amz-Bucket-Region"
|
||||
ServerInfo = "Server"
|
||||
RetryAfter = "Retry-After"
|
||||
Location = "Location"
|
||||
CacheControl = "Cache-Control"
|
||||
ContentDisposition = "Content-Disposition"
|
||||
Authorization = "Authorization"
|
||||
Action = "Action"
|
||||
)
|
||||
|
||||
// Standard S3 HTTP request constants
|
||||
const (
|
||||
IfModifiedSince = "If-Modified-Since"
|
||||
IfUnmodifiedSince = "If-Unmodified-Since"
|
||||
IfMatch = "If-Match"
|
||||
IfNoneMatch = "If-None-Match"
|
||||
|
||||
// S3 extensions
|
||||
AmzCopySourceIfModifiedSince = "x-amz-copy-source-if-modified-since"
|
||||
AmzCopySourceIfUnmodifiedSince = "x-amz-copy-source-if-unmodified-since"
|
||||
|
||||
AmzCopySourceIfNoneMatch = "x-amz-copy-source-if-none-match"
|
||||
AmzCopySourceIfMatch = "x-amz-copy-source-if-match"
|
||||
|
||||
AmzCopySource = "X-Amz-Copy-Source"
|
||||
AmzCopySourceVersionID = "X-Amz-Copy-Source-Version-Id"
|
||||
AmzCopySourceRange = "X-Amz-Copy-Source-Range"
|
||||
|
||||
// Signature V4 related contants.
|
||||
AmzContentSha256 = "X-Amz-Content-Sha256"
|
||||
AmzDate = "X-Amz-Date"
|
||||
AmzAlgorithm = "X-Amz-Algorithm"
|
||||
AmzExpires = "X-Amz-Expires"
|
||||
AmzSignedHeaders = "X-Amz-SignedHeaders"
|
||||
AmzSignature = "X-Amz-Signature"
|
||||
AmzCredential = "X-Amz-Credential"
|
||||
AmzSecurityToken = "X-Amz-Security-Token"
|
||||
AmzDecodedContentLength = "X-Amz-Decoded-Content-Length"
|
||||
|
||||
// Signature v2 related constants
|
||||
AmzSignatureV2 = "Signature"
|
||||
AmzAccessKeyID = "AWSAccessKeyId"
|
||||
|
||||
// Response request id.
|
||||
AmzRequestID = "x-amz-request-id"
|
||||
|
||||
// Deployment id.
|
||||
MinioDeploymentID = "x-minio-deployment-id"
|
||||
)
|
||||
@@ -1,115 +0,0 @@
|
||||
/*
|
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/pubsub"
|
||||
"github.com/minio/minio/pkg/trace"
|
||||
)
|
||||
|
||||
//HTTPTraceSys holds global trace state
|
||||
type HTTPTraceSys struct {
|
||||
peers []*peerRESTClient
|
||||
pubsub *pubsub.PubSub
|
||||
}
|
||||
|
||||
// NewTraceSys - creates new HTTPTraceSys with all nodes subscribed to
|
||||
// the trace pub sub system
|
||||
func NewTraceSys(ctx context.Context, endpoints EndpointList) *HTTPTraceSys {
|
||||
remoteHosts := getRemoteHosts(endpoints)
|
||||
remoteClients, err := getRestClients(remoteHosts)
|
||||
if err != nil {
|
||||
logger.FatalIf(err, "Unable to start httptrace sub system")
|
||||
}
|
||||
|
||||
ps := pubsub.New()
|
||||
return &HTTPTraceSys{
|
||||
remoteClients, ps,
|
||||
}
|
||||
}
|
||||
|
||||
// HasTraceListeners returns true if trace listeners are registered
|
||||
// for this node or peers
|
||||
func (sys *HTTPTraceSys) HasTraceListeners() bool {
|
||||
return sys != nil && sys.pubsub.HasSubscribers()
|
||||
}
|
||||
|
||||
// Publish - publishes trace message to the http trace pubsub system
|
||||
func (sys *HTTPTraceSys) Publish(traceMsg trace.Info) {
|
||||
sys.pubsub.Publish(traceMsg)
|
||||
}
|
||||
|
||||
// Trace writes http trace to writer
|
||||
func (sys *HTTPTraceSys) Trace(doneCh chan struct{}, trcAll bool) chan []byte {
|
||||
traceCh := make(chan []byte)
|
||||
go func() {
|
||||
defer close(traceCh)
|
||||
|
||||
var wg = &sync.WaitGroup{}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
ch := sys.pubsub.Subscribe()
|
||||
defer sys.pubsub.Unsubscribe(ch)
|
||||
for {
|
||||
select {
|
||||
case entry := <-ch:
|
||||
trcInfo := entry.(trace.Info)
|
||||
path := strings.TrimPrefix(trcInfo.ReqInfo.Path, "/")
|
||||
// omit inter-node traffic if trcAll is false
|
||||
if !trcAll && strings.HasPrefix(path, minioReservedBucket) {
|
||||
continue
|
||||
}
|
||||
buf.Reset()
|
||||
enc := json.NewEncoder(buf)
|
||||
enc.SetEscapeHTML(false)
|
||||
if err := enc.Encode(trcInfo); err != nil {
|
||||
continue
|
||||
}
|
||||
traceCh <- buf.Bytes()
|
||||
case <-doneCh:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
for _, peer := range sys.peers {
|
||||
wg.Add(1)
|
||||
go func(peer *peerRESTClient) {
|
||||
defer wg.Done()
|
||||
ch, err := peer.Trace(doneCh, trcAll)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for entry := range ch {
|
||||
traceCh <- entry
|
||||
}
|
||||
}(peer)
|
||||
}
|
||||
wg.Wait()
|
||||
}()
|
||||
return traceCh
|
||||
}
|
||||
+74
-2
@@ -17,6 +17,7 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"path"
|
||||
@@ -649,16 +650,87 @@ func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
|
||||
return cred, ok && cred.IsValid()
|
||||
}
|
||||
|
||||
// IsAllowed - checks given policy args is allowed to continue the Rest API.
|
||||
func (sys *IAMSys) IsAllowed(args iampolicy.Args) bool {
|
||||
// IsAllowedSTS is meant for STS based temporary credentials,
|
||||
// which implements claims validation and verification other than
|
||||
// applying policies.
|
||||
func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args) bool {
|
||||
pname, ok := args.Claims[iampolicy.PolicyName]
|
||||
if !ok {
|
||||
// When claims are set, it should have a "policy" field.
|
||||
return false
|
||||
}
|
||||
pnameStr, ok := pname.(string)
|
||||
if !ok {
|
||||
// When claims has "policy" field, it should be string.
|
||||
return false
|
||||
}
|
||||
|
||||
sys.RLock()
|
||||
defer sys.RUnlock()
|
||||
|
||||
// If policy is available for given user, check the policy.
|
||||
name, ok := sys.iamPolicyMap[args.AccountName]
|
||||
if !ok {
|
||||
// No policy available reject.
|
||||
return false
|
||||
}
|
||||
|
||||
if pnameStr != name {
|
||||
// When claims has a policy, it should match the
|
||||
// policy of args.AccountName which server remembers.
|
||||
// if not reject such requests.
|
||||
return false
|
||||
}
|
||||
|
||||
// Now check if we have a sessionPolicy.
|
||||
spolicy, ok := args.Claims[iampolicy.SessionPolicyName]
|
||||
if !ok {
|
||||
// Sub policy not set, this is most common since subPolicy
|
||||
// is optional, use the top level policy only.
|
||||
p, ok := sys.iamCannedPolicyMap[pnameStr]
|
||||
return ok && p.IsAllowed(args)
|
||||
}
|
||||
|
||||
spolicyStr, ok := spolicy.(string)
|
||||
if !ok {
|
||||
// Sub policy if set, should be a string reject
|
||||
// malformed/malicious requests.
|
||||
return false
|
||||
}
|
||||
|
||||
// Check if policy is parseable.
|
||||
subPolicy, err := iampolicy.ParseConfig(bytes.NewReader([]byte(spolicyStr)))
|
||||
if err != nil {
|
||||
// Log any error in input session policy config.
|
||||
logger.LogIf(context.Background(), err)
|
||||
return false
|
||||
}
|
||||
|
||||
// Policy without Version string value reject it.
|
||||
if subPolicy.Version == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
// Sub policy is set and valid.
|
||||
p, ok := sys.iamCannedPolicyMap[pnameStr]
|
||||
return ok && p.IsAllowed(args) && subPolicy.IsAllowed(args)
|
||||
}
|
||||
|
||||
// IsAllowed - checks given policy args is allowed to continue the Rest API.
|
||||
func (sys *IAMSys) IsAllowed(args iampolicy.Args) bool {
|
||||
// If opa is configured, use OPA always.
|
||||
if globalPolicyOPA != nil {
|
||||
return globalPolicyOPA.IsAllowed(args)
|
||||
}
|
||||
|
||||
// With claims set, we should do STS related checks and validation.
|
||||
if len(args.Claims) > 0 {
|
||||
return sys.IsAllowedSTS(args)
|
||||
}
|
||||
|
||||
sys.RLock()
|
||||
defer sys.RUnlock()
|
||||
|
||||
// If policy is available for given user, check the policy.
|
||||
if name, found := sys.iamPolicyMap[args.AccountName]; found {
|
||||
p, ok := sys.iamCannedPolicyMap[name]
|
||||
|
||||
+1
-1
@@ -21,7 +21,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/minio/dsync"
|
||||
"github.com/minio/dsync/v2"
|
||||
)
|
||||
|
||||
// lockRequesterInfo stores various info from the client for each lock that is requested.
|
||||
|
||||
@@ -28,7 +28,7 @@ import (
|
||||
|
||||
"net/url"
|
||||
|
||||
"github.com/minio/dsync"
|
||||
"github.com/minio/dsync/v2"
|
||||
"github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/cmd/rest"
|
||||
|
||||
@@ -19,7 +19,7 @@ package cmd
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/minio/dsync"
|
||||
"github.com/minio/dsync/v2"
|
||||
xnet "github.com/minio/minio/pkg/net"
|
||||
)
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/dsync"
|
||||
"github.com/minio/dsync/v2"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
xnet "github.com/minio/minio/pkg/net"
|
||||
)
|
||||
|
||||
+1
-1
@@ -63,6 +63,6 @@ func AuditLog(w http.ResponseWriter, r *http.Request, api string, reqClaims map[
|
||||
}
|
||||
// Send audit logs only to http targets.
|
||||
for _, t := range AuditTargets {
|
||||
_ = t.Send(audit.ToEntry(w, r, api, statusCode, reqClaims))
|
||||
_ = t.Send(audit.ToEntry(w, r, api, statusCode, reqClaims, globalDeploymentID))
|
||||
}
|
||||
}
|
||||
|
||||
+10
-1
@@ -55,6 +55,8 @@ const (
|
||||
|
||||
var trimStrings []string
|
||||
|
||||
var globalDeploymentID string
|
||||
|
||||
// TimeFormat - logging time format.
|
||||
const TimeFormat string = "15:04:05 MST 01/02/2006"
|
||||
|
||||
@@ -154,6 +156,11 @@ func uniqueEntries(paths []string) []string {
|
||||
return m.ToSlice()
|
||||
}
|
||||
|
||||
// SetDeploymentID -- Deployment Id from the main package is set here
|
||||
func SetDeploymentID(deploymentID string) {
|
||||
globalDeploymentID = deploymentID
|
||||
}
|
||||
|
||||
// Init sets the trimStrings to possible GOPATHs
|
||||
// and GOROOT directories. Also append github.com/minio/minio
|
||||
// This is done to clean up the filename, when stack trace is
|
||||
@@ -317,7 +324,9 @@ func logIf(ctx context.Context, err error) {
|
||||
|
||||
// Get the cause for the Error
|
||||
message := err.Error()
|
||||
|
||||
if req.DeploymentID == "" {
|
||||
req.DeploymentID = globalDeploymentID
|
||||
}
|
||||
entry := log.Entry{
|
||||
DeploymentID: req.DeploymentID,
|
||||
Level: ErrorLvl.String(),
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/pkg/handlers"
|
||||
)
|
||||
|
||||
@@ -50,7 +51,7 @@ type Entry struct {
|
||||
}
|
||||
|
||||
// ToEntry - constructs an audit entry object.
|
||||
func ToEntry(w http.ResponseWriter, r *http.Request, api string, statusCode int, reqClaims map[string]interface{}) Entry {
|
||||
func ToEntry(w http.ResponseWriter, r *http.Request, api string, statusCode int, reqClaims map[string]interface{}, deploymentID string) Entry {
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
object := vars["object"]
|
||||
@@ -67,13 +68,13 @@ func ToEntry(w http.ResponseWriter, r *http.Request, api string, statusCode int,
|
||||
for k, v := range w.Header() {
|
||||
respHeader[k] = strings.Join(v, ",")
|
||||
}
|
||||
respHeader["Etag"] = strings.Trim(respHeader["Etag"], `"`)
|
||||
respHeader[xhttp.ETag] = strings.Trim(respHeader[xhttp.ETag], `"`)
|
||||
|
||||
entry := Entry{
|
||||
Version: Version,
|
||||
DeploymentID: w.Header().Get("x-minio-deployment-id"),
|
||||
DeploymentID: deploymentID,
|
||||
RemoteHost: handlers.GetSourceIP(r),
|
||||
RequestID: w.Header().Get("x-amz-request-id"),
|
||||
RequestID: w.Header().Get(xhttp.AmzRequestID),
|
||||
UserAgent: r.UserAgent(),
|
||||
Time: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
ReqQuery: reqQuery,
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
gohttp "net/http"
|
||||
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
@@ -49,11 +50,11 @@ func (h *Target) startHTTPLogger() {
|
||||
continue
|
||||
}
|
||||
|
||||
req, err := gohttp.NewRequest("POST", h.endpoint, bytes.NewBuffer(logJSON))
|
||||
req, err := gohttp.NewRequest(http.MethodPost, h.endpoint, bytes.NewBuffer(logJSON))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set(xhttp.ContentType, "application/json")
|
||||
|
||||
resp, err := h.client.Do(req)
|
||||
if err != nil {
|
||||
|
||||
+21
-1
@@ -34,11 +34,25 @@ var (
|
||||
},
|
||||
[]string{"request_type"},
|
||||
)
|
||||
minioVersionInfo = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: "minio",
|
||||
Name: "version_info",
|
||||
Help: "Version of current MinIO server instance",
|
||||
},
|
||||
[]string{
|
||||
// current version
|
||||
"version",
|
||||
// commit-id of the current version
|
||||
"commit",
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
func init() {
|
||||
prometheus.MustRegister(httpRequestsDuration)
|
||||
prometheus.MustRegister(newMinioCollector())
|
||||
prometheus.MustRegister(minioVersionInfo)
|
||||
}
|
||||
|
||||
// newMinioCollector describes the collector
|
||||
@@ -64,6 +78,9 @@ func (c *minioCollector) Describe(ch chan<- *prometheus.Desc) {
|
||||
// Collect is called by the Prometheus registry when collecting metrics.
|
||||
func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
|
||||
|
||||
// Expose MinIO's version information
|
||||
minioVersionInfo.WithLabelValues(Version, CommitID).Add(1)
|
||||
|
||||
// Always expose network stats
|
||||
|
||||
// Network Sent/Received Bytes
|
||||
@@ -184,7 +201,10 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
|
||||
func metricsHandler() http.Handler {
|
||||
registry := prometheus.NewRegistry()
|
||||
|
||||
err := registry.Register(httpRequestsDuration)
|
||||
err := registry.Register(minioVersionInfo)
|
||||
logger.LogIf(context.Background(), err)
|
||||
|
||||
err = registry.Register(httpRequestsDuration)
|
||||
logger.LogIf(context.Background(), err)
|
||||
|
||||
err = registry.Register(newMinioCollector())
|
||||
|
||||
+12
-11
@@ -27,7 +27,7 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/minio/dsync"
|
||||
"github.com/minio/dsync/v2"
|
||||
"github.com/minio/lsync"
|
||||
"github.com/minio/minio-go/v6/pkg/set"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
@@ -127,7 +127,7 @@ type nsLockMap struct {
|
||||
}
|
||||
|
||||
// Lock the namespace resource.
|
||||
func (n *nsLockMap) lock(volume, path string, lockSource, opsID string, readLock bool, timeout time.Duration) (locked bool) {
|
||||
func (n *nsLockMap) lock(ctx context.Context, volume, path string, lockSource, opsID string, readLock bool, timeout time.Duration) (locked bool) {
|
||||
var nsLk *nsLock
|
||||
|
||||
n.lockMapMutex.Lock()
|
||||
@@ -135,7 +135,7 @@ func (n *nsLockMap) lock(volume, path string, lockSource, opsID string, readLock
|
||||
nsLk, found := n.lockMap[param]
|
||||
if !found {
|
||||
n.lockMap[param] = &nsLock{
|
||||
LRWMutex: &lsync.LRWMutex{},
|
||||
LRWMutex: lsync.NewLRWMutex(ctx),
|
||||
ref: 1,
|
||||
}
|
||||
nsLk = n.lockMap[param]
|
||||
@@ -199,7 +199,7 @@ func (n *nsLockMap) Lock(volume, path, opsID string, timeout time.Duration) (loc
|
||||
readLock := false // This is a write lock.
|
||||
|
||||
lockSource := getSource() // Useful for debugging
|
||||
return n.lock(volume, path, lockSource, opsID, readLock, timeout)
|
||||
return n.lock(context.Background(), volume, path, lockSource, opsID, readLock, timeout)
|
||||
}
|
||||
|
||||
// Unlock - unlocks any previously acquired write locks.
|
||||
@@ -213,7 +213,7 @@ func (n *nsLockMap) RLock(volume, path, opsID string, timeout time.Duration) (lo
|
||||
readLock := true
|
||||
|
||||
lockSource := getSource() // Useful for debugging
|
||||
return n.lock(volume, path, lockSource, opsID, readLock, timeout)
|
||||
return n.lock(context.Background(), volume, path, lockSource, opsID, readLock, timeout)
|
||||
}
|
||||
|
||||
// RUnlock - unlocks any previously acquired read locks.
|
||||
@@ -241,7 +241,7 @@ func (n *nsLockMap) ForceUnlock(volume, path string) {
|
||||
// are blocking can now proceed as normal and any new locks will also
|
||||
// participate normally.
|
||||
if n.isDistXL { // For distributed mode, broadcast ForceUnlock message.
|
||||
dsync.NewDRWMutex(pathJoin(volume, path), globalDsync).ForceUnlock()
|
||||
dsync.NewDRWMutex(context.Background(), pathJoin(volume, path), globalDsync).ForceUnlock()
|
||||
}
|
||||
|
||||
// Remove lock from the map.
|
||||
@@ -291,6 +291,7 @@ func (di *distLockInstance) RUnlock() {
|
||||
|
||||
// localLockInstance - frontend/top-level interface for namespace locks.
|
||||
type localLockInstance struct {
|
||||
ctx context.Context
|
||||
ns *nsLockMap
|
||||
volume, path, opsID string
|
||||
}
|
||||
@@ -298,12 +299,12 @@ type localLockInstance struct {
|
||||
// NewNSLock - returns a lock instance for a given volume and
|
||||
// path. The returned lockInstance object encapsulates the nsLockMap,
|
||||
// volume, path and operation ID.
|
||||
func (n *nsLockMap) NewNSLock(volume, path string) RWLocker {
|
||||
func (n *nsLockMap) NewNSLock(ctx context.Context, volume, path string) RWLocker {
|
||||
opsID := mustGetUUID()
|
||||
if n.isDistXL {
|
||||
return &distLockInstance{dsync.NewDRWMutex(pathJoin(volume, path), globalDsync), volume, path, opsID}
|
||||
return &distLockInstance{dsync.NewDRWMutex(ctx, pathJoin(volume, path), globalDsync), volume, path, opsID}
|
||||
}
|
||||
return &localLockInstance{n, volume, path, opsID}
|
||||
return &localLockInstance{ctx, n, volume, path, opsID}
|
||||
}
|
||||
|
||||
// Lock - block until write lock is taken or timeout has occurred.
|
||||
@@ -311,7 +312,7 @@ func (li *localLockInstance) GetLock(timeout *dynamicTimeout) (timedOutErr error
|
||||
lockSource := getSource()
|
||||
start := UTCNow()
|
||||
readLock := false
|
||||
if !li.ns.lock(li.volume, li.path, lockSource, li.opsID, readLock, timeout.Timeout()) {
|
||||
if !li.ns.lock(li.ctx, li.volume, li.path, lockSource, li.opsID, readLock, timeout.Timeout()) {
|
||||
timeout.LogFailure()
|
||||
return OperationTimedOut{Path: li.path}
|
||||
}
|
||||
@@ -330,7 +331,7 @@ func (li *localLockInstance) GetRLock(timeout *dynamicTimeout) (timedOutErr erro
|
||||
lockSource := getSource()
|
||||
start := UTCNow()
|
||||
readLock := true
|
||||
if !li.ns.lock(li.volume, li.path, lockSource, li.opsID, readLock, timeout.Timeout()) {
|
||||
if !li.ns.lock(li.ctx, li.volume, li.path, lockSource, li.opsID, readLock, timeout.Timeout()) {
|
||||
timeout.LogFailure()
|
||||
return OperationTimedOut{Path: li.path}
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -31,7 +32,7 @@ func TestGetSource(t *testing.T) {
|
||||
currentSource := func() string { return getSource() }
|
||||
gotSource := currentSource()
|
||||
// Hard coded line number, 32, in the "expectedSource" value
|
||||
expectedSource := "[namespace-lock_test.go:32:TestGetSource()]"
|
||||
expectedSource := "[namespace-lock_test.go:33:TestGetSource()]"
|
||||
if gotSource != expectedSource {
|
||||
t.Errorf("expected : %s, got : %s", expectedSource, gotSource)
|
||||
}
|
||||
@@ -204,7 +205,7 @@ func TestNamespaceForceUnlockTest(t *testing.T) {
|
||||
isDistXL := false
|
||||
initNSLock(isDistXL)
|
||||
// Create lock.
|
||||
lock := globalNSMutex.NewNSLock("bucket", "object")
|
||||
lock := globalNSMutex.NewNSLock(context.Background(), "bucket", "object")
|
||||
if lock.GetLock(newDynamicTimeout(60*time.Second, time.Second)) != nil {
|
||||
t.Fatalf("Failed to get lock")
|
||||
}
|
||||
@@ -215,7 +216,7 @@ func TestNamespaceForceUnlockTest(t *testing.T) {
|
||||
|
||||
go func() {
|
||||
// Try to claim lock again.
|
||||
anotherLock := globalNSMutex.NewNSLock("bucket", "object")
|
||||
anotherLock := globalNSMutex.NewNSLock(context.Background(), "bucket", "object")
|
||||
if anotherLock.GetLock(newDynamicTimeout(60*time.Second, time.Second)) != nil {
|
||||
t.Errorf("Failed to get lock")
|
||||
return
|
||||
|
||||
@@ -195,3 +195,10 @@ func (d *naughtyDisk) ReadAll(volume string, path string) (buf []byte, err error
|
||||
}
|
||||
return d.disk.ReadAll(volume, path)
|
||||
}
|
||||
|
||||
func (d *naughtyDisk) VerifyFile(volume, path string, algo BitrotAlgorithm, sum []byte, shardSize int64) error {
|
||||
if err := d.calcError(); err != nil {
|
||||
return err
|
||||
}
|
||||
return d.disk.VerifyFile(volume, path, algo, sum, shardSize)
|
||||
}
|
||||
|
||||
+3
-3
@@ -190,10 +190,10 @@ func isHostIP(ipAddress string) bool {
|
||||
return net.ParseIP(host) != nil
|
||||
}
|
||||
|
||||
// checkPortAvailability - check if given port is already in use.
|
||||
// checkPortAvailability - check if given host and port is already in use.
|
||||
// Note: The check method tries to listen on given port and closes it.
|
||||
// It is possible to have a disconnected client in this tiny window of time.
|
||||
func checkPortAvailability(port string) (err error) {
|
||||
func checkPortAvailability(host, port string) (err error) {
|
||||
// Return true if err is "address already in use" error.
|
||||
isAddrInUseErr := func(err error) (b bool) {
|
||||
if opErr, ok := err.(*net.OpError); ok {
|
||||
@@ -209,7 +209,7 @@ func checkPortAvailability(port string) (err error) {
|
||||
|
||||
network := []string{"tcp", "tcp4", "tcp6"}
|
||||
for _, n := range network {
|
||||
l, err := net.Listen(n, net.JoinHostPort("", port))
|
||||
l, err := net.Listen(n, net.JoinHostPort(host, port))
|
||||
if err == nil {
|
||||
// As we are able to listen on this network, the port is not in use.
|
||||
// Close the listener and continue check other networks.
|
||||
|
||||
+5
-3
@@ -206,11 +206,13 @@ func TestCheckPortAvailability(t *testing.T) {
|
||||
defer listener.Close()
|
||||
|
||||
testCases := []struct {
|
||||
host string
|
||||
port string
|
||||
expectedErr error
|
||||
}{
|
||||
{port, fmt.Errorf("listen tcp :%v: bind: address already in use", port)},
|
||||
{getFreePort(), nil},
|
||||
{"", port, fmt.Errorf("listen tcp :%v: bind: address already in use", port)},
|
||||
{"127.0.0.1", port, fmt.Errorf("listen tcp 127.0.0.1:%v: bind: address already in use", port)},
|
||||
{"", getFreePort(), nil},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
@@ -219,7 +221,7 @@ func TestCheckPortAvailability(t *testing.T) {
|
||||
continue
|
||||
}
|
||||
|
||||
err := checkPortAvailability(testCase.port)
|
||||
err := checkPortAvailability(testCase.host, testCase.port)
|
||||
if testCase.expectedErr == nil {
|
||||
if err != nil {
|
||||
t.Fatalf("error: expected = <nil>, got = %v", err)
|
||||
|
||||
+23
-3
@@ -34,6 +34,7 @@ import (
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/event"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
xnet "github.com/minio/minio/pkg/net"
|
||||
"github.com/minio/minio/pkg/policy"
|
||||
)
|
||||
@@ -230,6 +231,24 @@ func (sys *NotificationSys) LoadUsers() []NotificationPeerErr {
|
||||
return ng.Wait()
|
||||
}
|
||||
|
||||
// BackgroundHealStatus - returns background heal status of all peers
|
||||
func (sys *NotificationSys) BackgroundHealStatus() []madmin.BgHealState {
|
||||
states := make([]madmin.BgHealState, len(sys.peerClients))
|
||||
for idx, client := range sys.peerClients {
|
||||
if client == nil {
|
||||
continue
|
||||
}
|
||||
st, err := client.BackgroundHealStatus()
|
||||
if err != nil {
|
||||
logger.LogIf(context.Background(), err)
|
||||
} else {
|
||||
states[idx] = st
|
||||
}
|
||||
}
|
||||
|
||||
return states
|
||||
}
|
||||
|
||||
// StartProfiling - start profiling on remote peers, by initiating a remote RPC.
|
||||
func (sys *NotificationSys) StartProfiling(profiler string) []NotificationPeerErr {
|
||||
ng := WithNPeers(len(sys.peerClients))
|
||||
@@ -583,7 +602,7 @@ func (sys *NotificationSys) initListeners(ctx context.Context, objAPI ObjectLaye
|
||||
// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
|
||||
// and configFile, take a transaction lock to avoid data race between readConfig()
|
||||
// and saveConfig().
|
||||
objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
|
||||
objLock := globalNSMutex.NewNSLock(ctx, minioMetaBucket, transactionConfigFile)
|
||||
if err := objLock.GetRLock(globalOperationTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -659,6 +678,7 @@ func (sys *NotificationSys) refresh(objAPI ObjectLayer) error {
|
||||
if _, ok := err.(*event.ErrARNNotFound); ok {
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
if err == errNoSuchNotifications {
|
||||
continue
|
||||
@@ -1038,7 +1058,7 @@ func SaveListener(objAPI ObjectLayer, bucketName string, eventNames []event.Name
|
||||
// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
|
||||
// and configFile, take a transaction lock to avoid data race between readConfig()
|
||||
// and saveConfig().
|
||||
objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
|
||||
objLock := globalNSMutex.NewNSLock(ctx, minioMetaBucket, transactionConfigFile)
|
||||
if err := objLock.GetLock(globalOperationTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1089,7 +1109,7 @@ func RemoveListener(objAPI ObjectLayer, bucketName string, targetID event.Target
|
||||
// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
|
||||
// and configFile, take a transaction lock to avoid data race between readConfig()
|
||||
// and saveConfig().
|
||||
objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
|
||||
objLock := globalNSMutex.NewNSLock(ctx, minioMetaBucket, transactionConfigFile)
|
||||
if err := objLock.GetLock(globalOperationTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -21,6 +21,8 @@ import (
|
||||
"path"
|
||||
"sync"
|
||||
|
||||
"strings"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
)
|
||||
@@ -234,7 +236,94 @@ func removeListenerConfig(ctx context.Context, objAPI ObjectLayer, bucket string
|
||||
return objAPI.DeleteObject(ctx, minioMetaBucket, lcPath)
|
||||
}
|
||||
|
||||
func listObjectsNonSlash(ctx context.Context, obj ObjectLayer, bucket, prefix, marker, delimiter string, maxKeys int, tpool *TreeWalkPool, listDir ListDirFunc, getObjInfo func(context.Context, string, string) (ObjectInfo, error), getObjectInfoDirs ...func(context.Context, string, string) (ObjectInfo, error)) (loi ListObjectsInfo, err error) {
|
||||
endWalkCh := make(chan struct{})
|
||||
defer close(endWalkCh)
|
||||
recursive := true
|
||||
walkResultCh := startTreeWalk(ctx, bucket, prefix, "", recursive, listDir, endWalkCh)
|
||||
|
||||
var objInfos []ObjectInfo
|
||||
var eof bool
|
||||
var prevPrefix string
|
||||
|
||||
for {
|
||||
if len(objInfos) == maxKeys {
|
||||
break
|
||||
}
|
||||
result, ok := <-walkResultCh
|
||||
if !ok {
|
||||
eof = true
|
||||
break
|
||||
}
|
||||
|
||||
var objInfo ObjectInfo
|
||||
var err error
|
||||
|
||||
index := strings.Index(strings.TrimPrefix(result.entry, prefix), delimiter)
|
||||
if index == -1 {
|
||||
objInfo, err = getObjInfo(ctx, bucket, result.entry)
|
||||
if err != nil {
|
||||
// Ignore errFileNotFound as the object might have got
|
||||
// deleted in the interim period of listing and getObjectInfo(),
|
||||
// ignore quorum error as it might be an entry from an outdated disk.
|
||||
if IsErrIgnored(err, []error{
|
||||
errFileNotFound,
|
||||
errXLReadQuorum,
|
||||
}...) {
|
||||
continue
|
||||
}
|
||||
return loi, toObjectErr(err, bucket, prefix)
|
||||
}
|
||||
} else {
|
||||
index = len(prefix) + index + len(delimiter)
|
||||
currPrefix := result.entry[:index]
|
||||
if currPrefix == prevPrefix {
|
||||
continue
|
||||
}
|
||||
prevPrefix = currPrefix
|
||||
|
||||
objInfo = ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: currPrefix,
|
||||
IsDir: true,
|
||||
}
|
||||
}
|
||||
|
||||
if objInfo.Name <= marker {
|
||||
continue
|
||||
}
|
||||
|
||||
objInfos = append(objInfos, objInfo)
|
||||
if result.end {
|
||||
eof = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
result := ListObjectsInfo{}
|
||||
for _, objInfo := range objInfos {
|
||||
if objInfo.IsDir {
|
||||
result.Prefixes = append(result.Prefixes, objInfo.Name)
|
||||
continue
|
||||
}
|
||||
result.Objects = append(result.Objects, objInfo)
|
||||
}
|
||||
|
||||
if !eof {
|
||||
result.IsTruncated = true
|
||||
if len(objInfos) > 0 {
|
||||
result.NextMarker = objInfos[len(objInfos)-1].Name
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func listObjects(ctx context.Context, obj ObjectLayer, bucket, prefix, marker, delimiter string, maxKeys int, tpool *TreeWalkPool, listDir ListDirFunc, getObjInfo func(context.Context, string, string) (ObjectInfo, error), getObjectInfoDirs ...func(context.Context, string, string) (ObjectInfo, error)) (loi ListObjectsInfo, err error) {
|
||||
if delimiter != slashSeparator && delimiter != "" {
|
||||
return listObjectsNonSlash(ctx, obj, bucket, prefix, marker, delimiter, maxKeys, tpool, listDir, getObjInfo, getObjectInfoDirs...)
|
||||
}
|
||||
|
||||
if err := checkListObjsArgs(ctx, bucket, prefix, marker, delimiter, obj); err != nil {
|
||||
return loi, err
|
||||
}
|
||||
|
||||
@@ -471,10 +471,6 @@ func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
|
||||
{"volatile-bucket-1", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
|
||||
{"volatile-bucket-2", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
|
||||
{"volatile-bucket-3", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false},
|
||||
// Valid, existing bucket, but sending invalid delimeter values (9-10).
|
||||
// Empty string < "" > and forward slash < / > are the ony two valid arguments for delimeter.
|
||||
{"test-bucket-list-object", "", "", "*", 0, ListObjectsInfo{}, fmt.Errorf("delimiter '%s' is not supported", "*"), false},
|
||||
{"test-bucket-list-object", "", "", "-", 0, ListObjectsInfo{}, fmt.Errorf("delimiter '%s' is not supported", "-"), false},
|
||||
// Testing for failure cases with both perfix and marker (11).
|
||||
// The prefix and marker combination to be valid it should satisfy strings.HasPrefix(marker, prefix).
|
||||
{"test-bucket-list-object", "asia", "europe-object", "", 0, ListObjectsInfo{}, fmt.Errorf("Invalid combination of marker '%s' and prefix '%s'", "europe-object", "asia"), false},
|
||||
|
||||
+17
-1
@@ -36,6 +36,7 @@ import (
|
||||
snappy "github.com/golang/snappy"
|
||||
"github.com/minio/minio-go/v6/pkg/s3utils"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/dns"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
@@ -346,7 +347,7 @@ func isCompressible(header http.Header, object string) bool {
|
||||
// Eliminate the non-compressible objects.
|
||||
func excludeForCompression(header http.Header, object string) bool {
|
||||
objStr := object
|
||||
contentType := header.Get("Content-Type")
|
||||
contentType := header.Get(xhttp.ContentType)
|
||||
if globalIsCompressionEnabled {
|
||||
// We strictly disable compression for standard extensions/content-types (`compressed`).
|
||||
if hasStringSuffixInSlice(objStr, standardExcludeCompressExtensions) || hasPattern(standardExcludeCompressContentTypes, contentType) {
|
||||
@@ -793,3 +794,18 @@ func (cr *snappyCompressReader) Read(p []byte) (int, error) {
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Returns error if the cancelCh has been closed (indicating that S3 client has disconnected)
|
||||
type detectDisconnect struct {
|
||||
io.ReadCloser
|
||||
cancelCh <-chan struct{}
|
||||
}
|
||||
|
||||
func (d *detectDisconnect) Read(p []byte) (int, error) {
|
||||
select {
|
||||
case <-d.cancelCh:
|
||||
return 0, io.ErrUnexpectedEOF
|
||||
default:
|
||||
return d.ReadCloser.Read(p)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/pkg/event"
|
||||
"github.com/minio/minio/pkg/handlers"
|
||||
)
|
||||
@@ -49,7 +50,7 @@ func checkCopyObjectPartPreconditions(ctx context.Context, w http.ResponseWriter
|
||||
// x-amz-copy-source-if-none-match
|
||||
func checkCopyObjectPreconditions(ctx context.Context, w http.ResponseWriter, r *http.Request, objInfo ObjectInfo, encETag string) bool {
|
||||
// Return false for methods other than GET and HEAD.
|
||||
if r.Method != "PUT" {
|
||||
if r.Method != http.MethodPut {
|
||||
return false
|
||||
}
|
||||
if encETag == "" {
|
||||
@@ -68,15 +69,15 @@ func checkCopyObjectPreconditions(ctx context.Context, w http.ResponseWriter, r
|
||||
setCommonHeaders(w)
|
||||
|
||||
// set object-related metadata headers
|
||||
w.Header().Set("Last-Modified", objInfo.ModTime.UTC().Format(http.TimeFormat))
|
||||
w.Header().Set(xhttp.LastModified, objInfo.ModTime.UTC().Format(http.TimeFormat))
|
||||
|
||||
if objInfo.ETag != "" {
|
||||
w.Header()["ETag"] = []string{"\"" + objInfo.ETag + "\""}
|
||||
w.Header()[xhttp.ETag] = []string{"\"" + objInfo.ETag + "\""}
|
||||
}
|
||||
}
|
||||
// x-amz-copy-source-if-modified-since: Return the object only if it has been modified
|
||||
// since the specified time otherwise return 412 (precondition failed).
|
||||
ifModifiedSinceHeader := r.Header.Get("x-amz-copy-source-if-modified-since")
|
||||
ifModifiedSinceHeader := r.Header.Get(xhttp.AmzCopySourceIfModifiedSince)
|
||||
if ifModifiedSinceHeader != "" {
|
||||
if givenTime, err := time.Parse(http.TimeFormat, ifModifiedSinceHeader); err == nil {
|
||||
if !ifModifiedSince(objInfo.ModTime, givenTime) {
|
||||
@@ -90,7 +91,7 @@ func checkCopyObjectPreconditions(ctx context.Context, w http.ResponseWriter, r
|
||||
|
||||
// x-amz-copy-source-if-unmodified-since : Return the object only if it has not been
|
||||
// modified since the specified time, otherwise return a 412 (precondition failed).
|
||||
ifUnmodifiedSinceHeader := r.Header.Get("x-amz-copy-source-if-unmodified-since")
|
||||
ifUnmodifiedSinceHeader := r.Header.Get(xhttp.AmzCopySourceIfUnmodifiedSince)
|
||||
if ifUnmodifiedSinceHeader != "" {
|
||||
if givenTime, err := time.Parse(http.TimeFormat, ifUnmodifiedSinceHeader); err == nil {
|
||||
if ifModifiedSince(objInfo.ModTime, givenTime) {
|
||||
@@ -106,7 +107,7 @@ func checkCopyObjectPreconditions(ctx context.Context, w http.ResponseWriter, r
|
||||
|
||||
// x-amz-copy-source-if-match : Return the object only if its entity tag (ETag) is the
|
||||
// same as the one specified; otherwise return a 412 (precondition failed).
|
||||
ifMatchETagHeader := r.Header.Get("x-amz-copy-source-if-match")
|
||||
ifMatchETagHeader := r.Header.Get(xhttp.AmzCopySourceIfMatch)
|
||||
if ifMatchETagHeader != "" {
|
||||
etag := objInfo.ETag
|
||||
if shouldDecryptEtag {
|
||||
@@ -122,7 +123,7 @@ func checkCopyObjectPreconditions(ctx context.Context, w http.ResponseWriter, r
|
||||
|
||||
// If-None-Match : Return the object only if its entity tag (ETag) is different from the
|
||||
// one specified otherwise, return a 304 (not modified).
|
||||
ifNoneMatchETagHeader := r.Header.Get("x-amz-copy-source-if-none-match")
|
||||
ifNoneMatchETagHeader := r.Header.Get(xhttp.AmzCopySourceIfNoneMatch)
|
||||
if ifNoneMatchETagHeader != "" {
|
||||
etag := objInfo.ETag
|
||||
if shouldDecryptEtag {
|
||||
@@ -147,7 +148,7 @@ func checkCopyObjectPreconditions(ctx context.Context, w http.ResponseWriter, r
|
||||
// If-None-Match
|
||||
func checkPreconditions(ctx context.Context, w http.ResponseWriter, r *http.Request, objInfo ObjectInfo) bool {
|
||||
// Return false for methods other than GET and HEAD.
|
||||
if r.Method != "GET" && r.Method != "HEAD" {
|
||||
if r.Method != http.MethodGet && r.Method != http.MethodHead {
|
||||
return false
|
||||
}
|
||||
// If the object doesn't have a modtime (IsZero), or the modtime
|
||||
@@ -163,15 +164,15 @@ func checkPreconditions(ctx context.Context, w http.ResponseWriter, r *http.Requ
|
||||
setCommonHeaders(w)
|
||||
|
||||
// set object-related metadata headers
|
||||
w.Header().Set("Last-Modified", objInfo.ModTime.UTC().Format(http.TimeFormat))
|
||||
w.Header().Set(xhttp.LastModified, objInfo.ModTime.UTC().Format(http.TimeFormat))
|
||||
|
||||
if objInfo.ETag != "" {
|
||||
w.Header()["ETag"] = []string{"\"" + objInfo.ETag + "\""}
|
||||
w.Header()[xhttp.ETag] = []string{"\"" + objInfo.ETag + "\""}
|
||||
}
|
||||
}
|
||||
// If-Modified-Since : Return the object only if it has been modified since the specified time,
|
||||
// otherwise return a 304 (not modified).
|
||||
ifModifiedSinceHeader := r.Header.Get("If-Modified-Since")
|
||||
ifModifiedSinceHeader := r.Header.Get(xhttp.IfModifiedSince)
|
||||
if ifModifiedSinceHeader != "" {
|
||||
if givenTime, err := time.Parse(http.TimeFormat, ifModifiedSinceHeader); err == nil {
|
||||
if !ifModifiedSince(objInfo.ModTime, givenTime) {
|
||||
@@ -185,7 +186,7 @@ func checkPreconditions(ctx context.Context, w http.ResponseWriter, r *http.Requ
|
||||
|
||||
// If-Unmodified-Since : Return the object only if it has not been modified since the specified
|
||||
// time, otherwise return a 412 (precondition failed).
|
||||
ifUnmodifiedSinceHeader := r.Header.Get("If-Unmodified-Since")
|
||||
ifUnmodifiedSinceHeader := r.Header.Get(xhttp.IfUnmodifiedSince)
|
||||
if ifUnmodifiedSinceHeader != "" {
|
||||
if givenTime, err := time.Parse(http.TimeFormat, ifUnmodifiedSinceHeader); err == nil {
|
||||
if ifModifiedSince(objInfo.ModTime, givenTime) {
|
||||
@@ -199,7 +200,7 @@ func checkPreconditions(ctx context.Context, w http.ResponseWriter, r *http.Requ
|
||||
|
||||
// If-Match : Return the object only if its entity tag (ETag) is the same as the one specified;
|
||||
// otherwise return a 412 (precondition failed).
|
||||
ifMatchETagHeader := r.Header.Get("If-Match")
|
||||
ifMatchETagHeader := r.Header.Get(xhttp.IfMatch)
|
||||
if ifMatchETagHeader != "" {
|
||||
if !isETagEqual(objInfo.ETag, ifMatchETagHeader) {
|
||||
// If the object ETag does not match with the specified ETag.
|
||||
@@ -211,7 +212,7 @@ func checkPreconditions(ctx context.Context, w http.ResponseWriter, r *http.Requ
|
||||
|
||||
// If-None-Match : Return the object only if its entity tag (ETag) is different from the
|
||||
// one specified otherwise, return a 304 (not modified).
|
||||
ifNoneMatchETagHeader := r.Header.Get("If-None-Match")
|
||||
ifNoneMatchETagHeader := r.Header.Get(xhttp.IfNoneMatch)
|
||||
if ifNoneMatchETagHeader != "" {
|
||||
if isETagEqual(objInfo.ETag, ifNoneMatchETagHeader) {
|
||||
// If the object ETag matches with the specified ETag.
|
||||
|
||||
+31
-27
@@ -36,6 +36,7 @@ import (
|
||||
miniogo "github.com/minio/minio-go/v6"
|
||||
"github.com/minio/minio-go/v6/pkg/encrypt"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/dns"
|
||||
"github.com/minio/minio/pkg/event"
|
||||
@@ -50,12 +51,12 @@ import (
|
||||
|
||||
// supportedHeadGetReqParams - supported request parameters for GET and HEAD presigned request.
|
||||
var supportedHeadGetReqParams = map[string]string{
|
||||
"response-expires": "Expires",
|
||||
"response-content-type": "Content-Type",
|
||||
"response-cache-control": "Cache-Control",
|
||||
"response-content-encoding": "Content-Encoding",
|
||||
"response-content-language": "Content-Language",
|
||||
"response-content-disposition": "Content-Disposition",
|
||||
"response-expires": xhttp.Expires,
|
||||
"response-content-type": xhttp.ContentType,
|
||||
"response-cache-control": xhttp.CacheControl,
|
||||
"response-content-encoding": xhttp.ContentEncoding,
|
||||
"response-content-language": xhttp.ContentLanguage,
|
||||
"response-content-disposition": xhttp.ContentDisposition,
|
||||
}
|
||||
|
||||
const (
|
||||
@@ -165,8 +166,8 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
|
||||
BucketName: bucket,
|
||||
Key: object,
|
||||
Resource: r.URL.Path,
|
||||
RequestID: w.Header().Get(responseRequestIDKey),
|
||||
HostID: w.Header().Get(responseDeploymentIDKey),
|
||||
RequestID: w.Header().Get(xhttp.AmzRequestID),
|
||||
HostID: globalDeploymentID,
|
||||
})
|
||||
writeResponse(w, serr.HTTPStatusCode(), encodedErrorResponse, mimeXML)
|
||||
} else {
|
||||
@@ -207,8 +208,8 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
|
||||
BucketName: bucket,
|
||||
Key: object,
|
||||
Resource: r.URL.Path,
|
||||
RequestID: w.Header().Get(responseRequestIDKey),
|
||||
HostID: w.Header().Get(responseDeploymentIDKey),
|
||||
RequestID: w.Header().Get(xhttp.AmzRequestID),
|
||||
HostID: globalDeploymentID,
|
||||
})
|
||||
writeResponse(w, serr.HTTPStatusCode(), encodedErrorResponse, mimeXML)
|
||||
} else {
|
||||
@@ -650,7 +651,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
// TODO: Reject requests where body/payload is present, for now we don't even read it.
|
||||
|
||||
// Read escaped copy source path to check for parameters.
|
||||
cpSrcPath := r.Header.Get("X-Amz-Copy-Source")
|
||||
cpSrcPath := r.Header.Get(xhttp.AmzCopySource)
|
||||
|
||||
// Check https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectVersioning.html
|
||||
// Regardless of whether you have enabled versioning, each object in your bucket
|
||||
@@ -668,7 +669,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
// Note that url.Parse does the unescaping
|
||||
cpSrcPath = u.Path
|
||||
}
|
||||
if vid := r.Header.Get("X-Amz-Copy-Source-Version-Id"); vid != "" {
|
||||
if vid := r.Header.Get(xhttp.AmzCopySourceVersionID); vid != "" {
|
||||
// Check if versionId header was added, if yes then check if
|
||||
// its non "null" value, we should error out since we do not support
|
||||
// any versions other than "null".
|
||||
@@ -1052,7 +1053,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
if crypto.S3KMS.IsRequested(r.Header) {
|
||||
if crypto.S3KMS.IsRequested(r.Header) && !api.AllowSSEKMS() {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r)) // SSE-KMS is not supported
|
||||
return
|
||||
}
|
||||
@@ -1064,6 +1065,9 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
|
||||
bucket := vars["bucket"]
|
||||
object := vars["object"]
|
||||
|
||||
// To detect if the client has disconnected.
|
||||
r.Body = &detectDisconnect{r.Body, r.Context().Done()}
|
||||
|
||||
// X-Amz-Copy-Source shouldn't be set for this call.
|
||||
if _, ok := r.Header["X-Amz-Copy-Source"]; ok {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopySource), r.URL, guessIsBrowserReq(r))
|
||||
@@ -1178,7 +1182,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
|
||||
// This request header needs to be set prior to setting ObjectOptions
|
||||
if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) {
|
||||
if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) && !crypto.S3KMS.IsRequested(r.Header) {
|
||||
r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
|
||||
}
|
||||
|
||||
@@ -1268,7 +1272,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
|
||||
} else if hasServerSideEncryptionHeader(r.Header) {
|
||||
etag = getDecryptedETag(r.Header, objInfo, false)
|
||||
}
|
||||
w.Header()["ETag"] = []string{"\"" + etag + "\""}
|
||||
w.Header()[xhttp.ETag] = []string{"\"" + etag + "\""}
|
||||
|
||||
if objectAPI.IsEncryptionSupported() {
|
||||
if crypto.IsEncrypted(objInfo.UserDefined) {
|
||||
@@ -1315,7 +1319,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
if crypto.S3KMS.IsRequested(r.Header) {
|
||||
if crypto.S3KMS.IsRequested(r.Header) && !api.AllowSSEKMS() {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r)) // SSE-KMS is not supported
|
||||
return
|
||||
}
|
||||
@@ -1333,7 +1337,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
|
||||
}
|
||||
|
||||
// This request header needs to be set prior to setting ObjectOptions
|
||||
if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) {
|
||||
if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) && !crypto.S3KMS.IsRequested(r.Header) {
|
||||
r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
|
||||
}
|
||||
|
||||
@@ -1450,7 +1454,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
|
||||
}
|
||||
|
||||
// Read escaped copy source path to check for parameters.
|
||||
cpSrcPath := r.Header.Get("X-Amz-Copy-Source")
|
||||
cpSrcPath := r.Header.Get(xhttp.AmzCopySource)
|
||||
|
||||
// Check https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectVersioning.html
|
||||
// Regardless of whether you have enabled versioning, each object in your bucket
|
||||
@@ -1468,7 +1472,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
|
||||
// Note that url.Parse does the unescaping
|
||||
cpSrcPath = u.Path
|
||||
}
|
||||
if vid := r.Header.Get("X-Amz-Copy-Source-Version-Id"); vid != "" {
|
||||
if vid := r.Header.Get(xhttp.AmzCopySourceVersionID); vid != "" {
|
||||
// Check if X-Amz-Copy-Source-Version-Id header was added, if yes then check if
|
||||
// its non "null" value, we should error out since we do not support
|
||||
// any versions other than "null".
|
||||
@@ -1537,7 +1541,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
|
||||
|
||||
// Get request range.
|
||||
var rs *HTTPRangeSpec
|
||||
rangeHeader := r.Header.Get("x-amz-copy-source-range")
|
||||
rangeHeader := r.Header.Get(xhttp.AmzCopySourceRange)
|
||||
if rangeHeader != "" {
|
||||
var parseRangeErr error
|
||||
if rs, parseRangeErr = parseCopyPartRangeSpec(rangeHeader); parseRangeErr != nil {
|
||||
@@ -1729,7 +1733,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
|
||||
object := vars["object"]
|
||||
|
||||
// X-Amz-Copy-Source shouldn't be set for this call.
|
||||
if _, ok := r.Header["X-Amz-Copy-Source"]; ok {
|
||||
if _, ok := r.Header[xhttp.AmzCopySource]; ok {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopySource), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
@@ -1747,7 +1751,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
|
||||
rAuthType := getRequestAuthType(r)
|
||||
// For auth type streaming signature, we need to gather a different content length.
|
||||
if rAuthType == authTypeStreamingSigned {
|
||||
if sizeStr, ok := r.Header["X-Amz-Decoded-Content-Length"]; ok {
|
||||
if sizeStr, ok := r.Header[xhttp.AmzDecodedContentLength]; ok {
|
||||
if sizeStr[0] == "" {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
@@ -1949,7 +1953,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
|
||||
if isEncrypted {
|
||||
etag = tryDecryptETag(objectEncryptionKey, partInfo.ETag, crypto.SSEC.IsRequested(r.Header))
|
||||
}
|
||||
w.Header()["ETag"] = []string{"\"" + etag + "\""}
|
||||
w.Header()[xhttp.ETag] = []string{"\"" + etag + "\""}
|
||||
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
@@ -2264,15 +2268,15 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
|
||||
case "SlowDown", "XMinioServerNotInitialized", "XMinioReadQuorum", "XMinioWriteQuorum":
|
||||
// Set retry-after header to indicate user-agents to retry request after 120secs.
|
||||
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
|
||||
w.Header().Set("Retry-After", "120")
|
||||
w.Header().Set(xhttp.RetryAfter, "120")
|
||||
}
|
||||
|
||||
// Generate error response.
|
||||
errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path,
|
||||
w.Header().Get(responseRequestIDKey), w.Header().Get(responseDeploymentIDKey))
|
||||
w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
|
||||
encodedErrorResponse, _ := xml.Marshal(errorResponse)
|
||||
setCommonHeaders(w)
|
||||
w.Header().Set("Content-Type", string(mimeXML))
|
||||
w.Header().Set(xhttp.ContentType, string(mimeXML))
|
||||
w.Write(encodedErrorResponse)
|
||||
w.(http.Flusher).Flush()
|
||||
}
|
||||
@@ -2307,7 +2311,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
|
||||
}
|
||||
|
||||
// Set etag.
|
||||
w.Header()["ETag"] = []string{"\"" + objInfo.ETag + "\""}
|
||||
w.Header()[xhttp.ETag] = []string{"\"" + objInfo.ETag + "\""}
|
||||
|
||||
// Write success response.
|
||||
writeSuccessResponseXML(w, encodedSuccessResponse)
|
||||
|
||||
+58
-38
@@ -17,7 +17,6 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
@@ -31,8 +30,10 @@ import (
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/cmd/rest"
|
||||
"github.com/minio/minio/pkg/event"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
xnet "github.com/minio/minio/pkg/net"
|
||||
"github.com/minio/minio/pkg/policy"
|
||||
trace "github.com/minio/minio/pkg/trace"
|
||||
)
|
||||
|
||||
// client to talk to peer Nodes.
|
||||
@@ -212,7 +213,7 @@ func (client *peerRESTClient) ReloadFormat(dryRun bool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListenBucketNotification - send listent bucket notification to peer nodes.
|
||||
// ListenBucketNotification - send listen bucket notification to peer nodes.
|
||||
func (client *peerRESTClient) ListenBucketNotification(bucket string, eventNames []event.Name,
|
||||
pattern string, targetID event.TargetID, addr xnet.Host) error {
|
||||
args := listenBucketNotificationReq{
|
||||
@@ -422,52 +423,71 @@ func (client *peerRESTClient) SignalService(sig serviceSignal) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Trace - send http trace request to peer nodes
|
||||
func (client *peerRESTClient) Trace(doneCh chan struct{}, trcAll bool) (chan []byte, error) {
|
||||
ch := make(chan []byte)
|
||||
func (client *peerRESTClient) BackgroundHealStatus() (madmin.BgHealState, error) {
|
||||
respBody, err := client.call(peerRESTMethodBackgroundHealStatus, nil, nil, -1)
|
||||
if err != nil {
|
||||
return madmin.BgHealState{}, err
|
||||
}
|
||||
defer http.DrainBody(respBody)
|
||||
|
||||
state := madmin.BgHealState{}
|
||||
err = gob.NewDecoder(respBody).Decode(&state)
|
||||
return state, err
|
||||
}
|
||||
|
||||
func (client *peerRESTClient) doTrace(traceCh chan interface{}, doneCh chan struct{}, trcAll bool) {
|
||||
values := make(url.Values)
|
||||
values.Set(peerRESTTraceAll, strconv.FormatBool(trcAll))
|
||||
|
||||
// To cancel the REST request in case doneCh gets closed.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
cancelCh := make(chan struct{})
|
||||
defer close(cancelCh)
|
||||
go func() {
|
||||
cleanupFn := func(cancel context.CancelFunc, ch chan []byte, respBody io.ReadCloser) {
|
||||
close(ch)
|
||||
if cancel != nil {
|
||||
cancel()
|
||||
}
|
||||
http.DrainBody(respBody)
|
||||
select {
|
||||
case <-doneCh:
|
||||
case <-cancelCh:
|
||||
// There was an error in the REST request.
|
||||
}
|
||||
cancel()
|
||||
}()
|
||||
|
||||
respBody, err := client.callWithContext(ctx, peerRESTMethodTrace, values, nil, -1)
|
||||
defer http.DrainBody(respBody)
|
||||
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
dec := gob.NewDecoder(respBody)
|
||||
for {
|
||||
var info trace.Info
|
||||
if err = dec.Decode(&info); err != nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case traceCh <- info:
|
||||
default:
|
||||
// Do not block on slow receivers.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Trace - send http trace request to peer nodes
|
||||
func (client *peerRESTClient) Trace(traceCh chan interface{}, doneCh chan struct{}, trcAll bool) {
|
||||
go func() {
|
||||
for {
|
||||
values := make(url.Values)
|
||||
values.Set(peerRESTTraceAll, strconv.FormatBool(trcAll))
|
||||
// get cancellation context to properly unsubscribe peers
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
respBody, err := client.callWithContext(ctx, peerRESTMethodTrace, values, nil, -1)
|
||||
if err != nil {
|
||||
//retry
|
||||
time.Sleep(5 * time.Second)
|
||||
select {
|
||||
case <-doneCh:
|
||||
cleanupFn(cancel, ch, respBody)
|
||||
return
|
||||
default:
|
||||
}
|
||||
continue
|
||||
}
|
||||
bio := bufio.NewScanner(respBody)
|
||||
go func() {
|
||||
<-doneCh
|
||||
cancel()
|
||||
}()
|
||||
// Unmarshal each line, returns marshaled values.
|
||||
for bio.Scan() {
|
||||
ch <- bio.Bytes()
|
||||
}
|
||||
client.doTrace(traceCh, doneCh, trcAll)
|
||||
select {
|
||||
case <-doneCh:
|
||||
cleanupFn(cancel, ch, respBody)
|
||||
return
|
||||
default:
|
||||
// There was error in the REST request, retry after sometime as probably the peer is down.
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
}
|
||||
}()
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
func getRemoteHosts(endpoints EndpointList) []*xnet.Host {
|
||||
|
||||
@@ -26,6 +26,7 @@ const (
|
||||
peerRESTMethodDrivePerfInfo = "driveperfinfo"
|
||||
peerRESTMethodDeleteBucket = "deletebucket"
|
||||
peerRESTMethodSignalService = "signalservice"
|
||||
peerRESTMethodBackgroundHealStatus = "backgroundhealstatus"
|
||||
peerRESTMethodGetLocks = "getlocks"
|
||||
peerRESTMethodBucketPolicyRemove = "removebucketpolicy"
|
||||
peerRESTMethodLoadUser = "loaduser"
|
||||
|
||||
+38
-22
@@ -19,7 +19,6 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"encoding/gob"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
@@ -29,6 +28,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/event"
|
||||
xnet "github.com/minio/minio/pkg/net"
|
||||
@@ -332,7 +332,7 @@ func (s *peerRESTServer) CPULoadInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
}
|
||||
|
||||
ctx := newContext(r, w, "CPULoadInfo")
|
||||
info := localEndpointsCPULoad(globalEndpoints)
|
||||
info := localEndpointsCPULoad(globalEndpoints, r)
|
||||
|
||||
defer w.(http.Flusher).Flush()
|
||||
logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
|
||||
@@ -346,7 +346,7 @@ func (s *peerRESTServer) DrivePerfInfoHandler(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
|
||||
ctx := newContext(r, w, "DrivePerfInfo")
|
||||
info := localEndpointsDrivePerf(globalEndpoints)
|
||||
info := localEndpointsDrivePerf(globalEndpoints, r)
|
||||
|
||||
defer w.(http.Flusher).Flush()
|
||||
logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
|
||||
@@ -359,7 +359,7 @@ func (s *peerRESTServer) MemUsageInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
ctx := newContext(r, w, "MemUsageInfo")
|
||||
info := localEndpointsMemUsage(globalEndpoints)
|
||||
info := localEndpointsMemUsage(globalEndpoints, r)
|
||||
|
||||
defer w.(http.Flusher).Flush()
|
||||
logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
|
||||
@@ -676,39 +676,54 @@ func (s *peerRESTServer) TraceHandler(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
trcAll := r.URL.Query().Get(peerRESTTraceAll) == "true"
|
||||
|
||||
w.Header().Set("Connection", "close")
|
||||
w.Header().Set(xhttp.Connection, "close")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.(http.Flusher).Flush()
|
||||
ch := globalTrace.pubsub.Subscribe()
|
||||
defer globalTrace.pubsub.Unsubscribe(ch)
|
||||
|
||||
enc := json.NewEncoder(w)
|
||||
enc.SetEscapeHTML(false)
|
||||
filter := func(entry interface{}) bool {
|
||||
if trcAll {
|
||||
return true
|
||||
}
|
||||
trcInfo := entry.(trace.Info)
|
||||
return !strings.HasPrefix(trcInfo.ReqInfo.Path, minioReservedBucketPath)
|
||||
}
|
||||
|
||||
doneCh := make(chan struct{})
|
||||
defer close(doneCh)
|
||||
|
||||
// Trace Publisher uses nonblocking publish and hence does not wait for slow subscribers.
|
||||
// Use buffered channel to take care of burst sends or slow w.Write()
|
||||
ch := make(chan interface{}, 2000)
|
||||
globalHTTPTrace.Subscribe(ch, doneCh, filter)
|
||||
|
||||
enc := gob.NewEncoder(w)
|
||||
for {
|
||||
select {
|
||||
case entry := <-ch:
|
||||
trcInfo := entry.(trace.Info)
|
||||
path := strings.TrimPrefix(trcInfo.ReqInfo.Path, "/")
|
||||
// omit inter-node traffic if trcAll is false
|
||||
if !trcAll && strings.HasPrefix(path, minioReservedBucket) {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := enc.Encode(trcInfo); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := w.Write([]byte("\n")); err != nil {
|
||||
if err := enc.Encode(entry); err != nil {
|
||||
return
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *peerRESTServer) BackgroundHealStatusHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.IsValid(w, r) {
|
||||
s.writeErrorResponse(w, errors.New("invalid request"))
|
||||
return
|
||||
}
|
||||
|
||||
ctx := newContext(r, w, "BackgroundHealStatus")
|
||||
|
||||
state := getLocalBackgroundHealStatus()
|
||||
|
||||
defer w.(http.Flusher).Flush()
|
||||
logger.LogIf(ctx, gob.NewEncoder(w).Encode(state))
|
||||
}
|
||||
|
||||
func (s *peerRESTServer) writeErrorResponse(w http.ResponseWriter, err error) {
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
w.Write([]byte(err.Error()))
|
||||
@@ -755,6 +770,7 @@ func registerPeerRESTHandlers(router *mux.Router) {
|
||||
subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodReloadFormat).HandlerFunc(httpTraceHdrs(server.ReloadFormatHandler)).Queries(restQueries(peerRESTDryRun)...)
|
||||
|
||||
subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodTrace).HandlerFunc(server.TraceHandler)
|
||||
subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodBackgroundHealStatus).HandlerFunc(server.BackgroundHealStatusHandler)
|
||||
|
||||
router.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler))
|
||||
}
|
||||
|
||||
+102
-1
@@ -958,7 +958,7 @@ func (s *posix) ReadFile(volume, path string, offset int64, buffer []byte, verif
|
||||
}
|
||||
|
||||
if !bytes.Equal(h.Sum(nil), verifier.sum) {
|
||||
return 0, hashMismatchError{hex.EncodeToString(verifier.sum), hex.EncodeToString(h.Sum(nil))}
|
||||
return 0, HashMismatchError{hex.EncodeToString(verifier.sum), hex.EncodeToString(h.Sum(nil))}
|
||||
}
|
||||
|
||||
return int64(len(buffer)), nil
|
||||
@@ -1525,3 +1525,104 @@ func (s *posix) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err e
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *posix) VerifyFile(volume, path string, algo BitrotAlgorithm, sum []byte, shardSize int64) (err error) {
|
||||
defer func() {
|
||||
if err == errFaultyDisk {
|
||||
atomic.AddInt32(&s.ioErrCount, 1)
|
||||
}
|
||||
}()
|
||||
|
||||
if atomic.LoadInt32(&s.ioErrCount) > maxAllowedIOError {
|
||||
return errFaultyDisk
|
||||
}
|
||||
|
||||
if err = s.checkDiskFound(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
volumeDir, err := s.getVolDir(volume)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Stat a volume entry.
|
||||
_, err = os.Stat((volumeDir))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return errVolumeNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Validate effective path length before reading.
|
||||
filePath := pathJoin(volumeDir, path)
|
||||
if err = checkPathLength((filePath)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Open the file for reading.
|
||||
file, err := os.Open((filePath))
|
||||
if err != nil {
|
||||
switch {
|
||||
case os.IsNotExist(err):
|
||||
return errFileNotFound
|
||||
case os.IsPermission(err):
|
||||
return errFileAccessDenied
|
||||
case isSysErrNotDir(err):
|
||||
return errFileAccessDenied
|
||||
case isSysErrIO(err):
|
||||
return errFaultyDisk
|
||||
default:
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Close the file descriptor.
|
||||
defer file.Close()
|
||||
|
||||
if algo != HighwayHash256S {
|
||||
bufp := s.pool.Get().(*[]byte)
|
||||
defer s.pool.Put(bufp)
|
||||
|
||||
h := algo.New()
|
||||
if _, err = io.CopyBuffer(h, file, *bufp); err != nil {
|
||||
return err
|
||||
}
|
||||
if !bytes.Equal(h.Sum(nil), sum) {
|
||||
return HashMismatchError{hex.EncodeToString(sum), hex.EncodeToString(h.Sum(nil))}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
buf := make([]byte, shardSize)
|
||||
h := algo.New()
|
||||
hashBuf := make([]byte, h.Size())
|
||||
fi, err := file.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
size := fi.Size()
|
||||
for {
|
||||
if size == 0 {
|
||||
return nil
|
||||
}
|
||||
h.Reset()
|
||||
n, err := file.Read(hashBuf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
size -= int64(n)
|
||||
if size < int64(len(buf)) {
|
||||
buf = buf[:size]
|
||||
}
|
||||
n, err = file.Read(buf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
size -= int64(n)
|
||||
h.Write(buf)
|
||||
if !bytes.Equal(h.Sum(nil), hashBuf) {
|
||||
return HashMismatchError{hex.EncodeToString(hashBuf), hex.EncodeToString(h.Sum(nil))}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+96
-5
@@ -1251,13 +1251,13 @@ var posixReadFileWithVerifyTests = []struct {
|
||||
{file: "myobject", offset: 25, length: 74, algorithm: SHA256, expError: nil}, // 1
|
||||
{file: "myobject", offset: 29, length: 70, algorithm: SHA256, expError: nil}, // 2
|
||||
{file: "myobject", offset: 100, length: 0, algorithm: SHA256, expError: nil}, // 3
|
||||
{file: "myobject", offset: 1, length: 120, algorithm: SHA256, expError: hashMismatchError{}}, // 4
|
||||
{file: "myobject", offset: 1, length: 120, algorithm: SHA256, expError: HashMismatchError{}}, // 4
|
||||
{file: "myobject", offset: 3, length: 1100, algorithm: SHA256, expError: nil}, // 5
|
||||
{file: "myobject", offset: 2, length: 100, algorithm: SHA256, expError: hashMismatchError{}}, // 6
|
||||
{file: "myobject", offset: 2, length: 100, algorithm: SHA256, expError: HashMismatchError{}}, // 6
|
||||
{file: "myobject", offset: 1000, length: 1001, algorithm: SHA256, expError: nil}, // 7
|
||||
{file: "myobject", offset: 0, length: 100, algorithm: BLAKE2b512, expError: hashMismatchError{}}, // 8
|
||||
{file: "myobject", offset: 0, length: 100, algorithm: BLAKE2b512, expError: HashMismatchError{}}, // 8
|
||||
{file: "myobject", offset: 25, length: 74, algorithm: BLAKE2b512, expError: nil}, // 9
|
||||
{file: "myobject", offset: 29, length: 70, algorithm: BLAKE2b512, expError: hashMismatchError{}}, // 10
|
||||
{file: "myobject", offset: 29, length: 70, algorithm: BLAKE2b512, expError: HashMismatchError{}}, // 10
|
||||
{file: "myobject", offset: 100, length: 0, algorithm: BLAKE2b512, expError: nil}, // 11
|
||||
{file: "myobject", offset: 1, length: 120, algorithm: BLAKE2b512, expError: nil}, // 12
|
||||
{file: "myobject", offset: 3, length: 1100, algorithm: BLAKE2b512, expError: nil}, // 13
|
||||
@@ -1296,7 +1296,7 @@ func TestPosixReadFileWithVerify(t *testing.T) {
|
||||
if test.expError != nil {
|
||||
expected := h.Sum(nil)
|
||||
h.Write([]byte{0})
|
||||
test.expError = hashMismatchError{hex.EncodeToString(h.Sum(nil)), hex.EncodeToString(expected)}
|
||||
test.expError = HashMismatchError{hex.EncodeToString(h.Sum(nil)), hex.EncodeToString(expected)}
|
||||
}
|
||||
|
||||
buffer := make([]byte, test.length)
|
||||
@@ -1763,6 +1763,97 @@ func TestPosixStatFile(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Test posix.VerifyFile()
|
||||
func TestPosixVerifyFile(t *testing.T) {
|
||||
// We test 4 cases:
|
||||
// 1) Whole-file bitrot check on proper file
|
||||
// 2) Whole-file bitrot check on corrupted file
|
||||
// 3) Streaming bitrot check on proper file
|
||||
// 4) Streaming bitrot check on corrupted file
|
||||
|
||||
// create posix test setup
|
||||
posixStorage, path, err := newPosixTestSetup()
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to create posix test setup, %s", err)
|
||||
}
|
||||
defer os.RemoveAll(path)
|
||||
|
||||
volName := "testvol"
|
||||
fileName := "testfile"
|
||||
if err := posixStorage.MakeVol(volName); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// 1) Whole-file bitrot check on proper file
|
||||
size := int64(4*1024*1024 + 100*1024) // 4.1 MB
|
||||
data := make([]byte, size)
|
||||
if _, err := rand.Read(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
algo := HighwayHash256
|
||||
h := algo.New()
|
||||
h.Write(data)
|
||||
hashBytes := h.Sum(nil)
|
||||
if err := posixStorage.WriteAll(volName, fileName, bytes.NewBuffer(data)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := posixStorage.VerifyFile(volName, fileName, algo, hashBytes, 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// 2) Whole-file bitrot check on corrupted file
|
||||
if err := posixStorage.AppendFile(volName, fileName, []byte("a")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := posixStorage.VerifyFile(volName, fileName, algo, hashBytes, 0); err == nil {
|
||||
t.Fatal("expected to fail bitrot check")
|
||||
}
|
||||
|
||||
if err := posixStorage.DeleteFile(volName, fileName); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// 3) Streaming bitrot check on proper file
|
||||
algo = HighwayHash256S
|
||||
shardSize := int64(1024 * 1024)
|
||||
shard := make([]byte, shardSize)
|
||||
w := newStreamingBitrotWriter(posixStorage, volName, fileName, size, algo, shardSize)
|
||||
reader := bytes.NewReader(data)
|
||||
for {
|
||||
// Using io.CopyBuffer instead of this loop will not work for us as io.CopyBuffer
|
||||
// will use bytes.Buffer.ReadFrom() which will not do shardSize'ed writes causing error.
|
||||
n, err := reader.Read(shard)
|
||||
w.Write(shard[:n])
|
||||
if err == nil {
|
||||
continue
|
||||
}
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
w.Close()
|
||||
if err := posixStorage.VerifyFile(volName, fileName, algo, nil, shardSize); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// 4) Streaming bitrot check on corrupted file
|
||||
filePath := pathJoin(posixStorage.String(), volName, fileName)
|
||||
f, err := os.OpenFile(filePath, os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := f.WriteString("a"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.Close()
|
||||
if err := posixStorage.VerifyFile(volName, fileName, algo, nil, shardSize); err == nil {
|
||||
t.Fatal("expected to fail bitrot check")
|
||||
}
|
||||
}
|
||||
|
||||
// Checks for restrictions for min total disk space and inodes.
|
||||
func TestCheckDiskTotalMin(t *testing.T) {
|
||||
testCases := []struct {
|
||||
|
||||
+4
-3
@@ -48,7 +48,7 @@ func registerDistXLRouters(router *mux.Router, endpoints EndpointList) {
|
||||
|
||||
// List of some generic handlers which are applied for all incoming requests.
|
||||
var globalHandlers = []HandlerFunc{
|
||||
// set x-amz-request-id, x-minio-deployment-id header.
|
||||
// set x-amz-request-id header.
|
||||
addCustomHeaders,
|
||||
// set HTTP security headers such as Content-Security-Policy.
|
||||
addSecurityHeaders,
|
||||
@@ -119,8 +119,9 @@ func configureServerHandler(endpoints EndpointList) (http.Handler, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// Add API router, additionally all server mode support encryption.
|
||||
registerAPIRouter(router, true)
|
||||
// Add API router, additionally all server mode support encryption
|
||||
// but don't allow SSE-KMS.
|
||||
registerAPIRouter(router, true, false)
|
||||
|
||||
// Register rest of the handlers.
|
||||
return registerHandlers(router, globalHandlers...), nil
|
||||
|
||||
+5
-5
@@ -18,6 +18,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/gob"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
@@ -27,7 +28,7 @@ import (
|
||||
"syscall"
|
||||
|
||||
"github.com/minio/cli"
|
||||
"github.com/minio/dsync"
|
||||
"github.com/minio/dsync/v2"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/certs"
|
||||
@@ -36,6 +37,7 @@ import (
|
||||
func init() {
|
||||
logger.Init(GOPATH, GOROOT)
|
||||
logger.RegisterUIError(fmtError)
|
||||
gob.Register(HashMismatchError{})
|
||||
}
|
||||
|
||||
// ServerFlags - server command specific flags
|
||||
@@ -176,7 +178,7 @@ func serverHandleCmdArgs(ctx *cli.Context) {
|
||||
// to IPv6 address ie minio will start listening on IPv6 address whereas another
|
||||
// (non-)minio process is listening on IPv4 of given port.
|
||||
// To avoid this error sutiation we check for port availability.
|
||||
logger.FatalIf(checkPortAvailability(globalMinioPort), "Unable to start the server")
|
||||
logger.FatalIf(checkPortAvailability(globalMinioHost, globalMinioPort), "Unable to start the server")
|
||||
|
||||
globalIsXL = (setupType == XLSetupType)
|
||||
globalIsDistXL = (setupType == DistXLSetupType)
|
||||
@@ -294,9 +296,6 @@ func serverMain(ctx *cli.Context) {
|
||||
globalSweepHealState = initHealState()
|
||||
}
|
||||
|
||||
// initialize globalTrace system
|
||||
globalTrace = NewTraceSys(context.Background(), globalEndpoints)
|
||||
|
||||
// Configure server.
|
||||
var handler http.Handler
|
||||
handler, err = configureServerHandler(globalEndpoints)
|
||||
@@ -317,6 +316,7 @@ func serverMain(ctx *cli.Context) {
|
||||
}()
|
||||
|
||||
newObject, err := newObjectLayer(globalEndpoints)
|
||||
logger.SetDeploymentID(globalDeploymentID)
|
||||
if err != nil {
|
||||
// Stop watching for any certificate changes.
|
||||
globalTLSCerts.Stop()
|
||||
|
||||
+3
-2
@@ -34,6 +34,7 @@ import (
|
||||
"time"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/pkg/policy"
|
||||
)
|
||||
|
||||
@@ -1202,7 +1203,7 @@ func (s *TestSuiteCommon) TestPutObject(c *check) {
|
||||
c.Assert(err, nil)
|
||||
c.Assert(response.StatusCode, http.StatusOK)
|
||||
// The response Etag header should contain Md5sum of an empty string.
|
||||
c.Assert(response.Header.Get("Etag"), "\""+emptyETag+"\"")
|
||||
c.Assert(response.Header.Get(xhttp.ETag), "\""+emptyETag+"\"")
|
||||
}
|
||||
|
||||
// TestListBuckets - Make request for listing of all buckets.
|
||||
@@ -2728,5 +2729,5 @@ func (s *TestSuiteCommon) TestObjectMultipart(c *check) {
|
||||
parts = append(parts, part)
|
||||
}
|
||||
etag := getCompleteMultipartMD5(parts)
|
||||
c.Assert(canonicalizeETag(response.Header.Get("Etag")), etag)
|
||||
c.Assert(canonicalizeETag(response.Header.Get(xhttp.ETag)), etag)
|
||||
}
|
||||
|
||||
+14
-12
@@ -28,6 +28,8 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
)
|
||||
|
||||
@@ -68,13 +70,13 @@ var resourceList = []string{
|
||||
|
||||
func doesPolicySignatureV2Match(formValues http.Header) APIErrorCode {
|
||||
cred := globalServerConfig.GetCredential()
|
||||
accessKey := formValues.Get("AWSAccessKeyId")
|
||||
accessKey := formValues.Get(xhttp.AmzAccessKeyID)
|
||||
cred, _, s3Err := checkKeyValid(accessKey)
|
||||
if s3Err != ErrNone {
|
||||
return s3Err
|
||||
}
|
||||
policy := formValues.Get("Policy")
|
||||
signature := formValues.Get("Signature")
|
||||
signature := formValues.Get(xhttp.AmzSignatureV2)
|
||||
if !compareSignatureV2(signature, calculateSignatureV2(policy, cred.SecretKey)) {
|
||||
return ErrSignatureDoesNotMatch
|
||||
}
|
||||
@@ -131,11 +133,11 @@ func doesPresignV2SignatureMatch(r *http.Request) APIErrorCode {
|
||||
return ErrInvalidQueryParams
|
||||
}
|
||||
switch keyval[0] {
|
||||
case "AWSAccessKeyId":
|
||||
case xhttp.AmzAccessKeyID:
|
||||
accessKey = keyval[1]
|
||||
case "Signature":
|
||||
case xhttp.AmzSignatureV2:
|
||||
gotSignature = keyval[1]
|
||||
case "Expires":
|
||||
case xhttp.Expires:
|
||||
expires = keyval[1]
|
||||
default:
|
||||
filteredQueries = append(filteredQueries, query)
|
||||
@@ -177,13 +179,13 @@ func doesPresignV2SignatureMatch(r *http.Request) APIErrorCode {
|
||||
}
|
||||
|
||||
func getReqAccessKeyV2(r *http.Request) (auth.Credentials, bool, APIErrorCode) {
|
||||
if accessKey := r.URL.Query().Get("AWSAccessKeyId"); accessKey != "" {
|
||||
if accessKey := r.URL.Query().Get(xhttp.AmzAccessKeyID); accessKey != "" {
|
||||
return checkKeyValid(accessKey)
|
||||
}
|
||||
|
||||
// below is V2 Signed Auth header format, splitting on `space` (after the `AWS` string).
|
||||
// Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature
|
||||
authFields := strings.Split(r.Header.Get("Authorization"), " ")
|
||||
authFields := strings.Split(r.Header.Get(xhttp.Authorization), " ")
|
||||
if len(authFields) != 2 {
|
||||
return auth.Credentials{}, false, ErrMissingFields
|
||||
}
|
||||
@@ -219,7 +221,7 @@ func getReqAccessKeyV2(r *http.Request) (auth.Credentials, bool, APIErrorCode) {
|
||||
|
||||
func validateV2AuthHeader(r *http.Request) (auth.Credentials, APIErrorCode) {
|
||||
var cred auth.Credentials
|
||||
v2Auth := r.Header.Get("Authorization")
|
||||
v2Auth := r.Header.Get(xhttp.Authorization)
|
||||
if v2Auth == "" {
|
||||
return cred, ErrAuthHeaderEmpty
|
||||
}
|
||||
@@ -238,7 +240,7 @@ func validateV2AuthHeader(r *http.Request) (auth.Credentials, APIErrorCode) {
|
||||
}
|
||||
|
||||
func doesSignV2Match(r *http.Request) APIErrorCode {
|
||||
v2Auth := r.Header.Get("Authorization")
|
||||
v2Auth := r.Header.Get(xhttp.Authorization)
|
||||
cred, apiError := validateV2AuthHeader(r)
|
||||
if apiError != ErrNone {
|
||||
return apiError
|
||||
@@ -380,7 +382,7 @@ func getStringToSignV2(method string, encodedResource, encodedQuery string, head
|
||||
date := expires // Date is set to expires date for presign operations.
|
||||
if date == "" {
|
||||
// If expires date is empty then request header Date is used.
|
||||
date = headers.Get("Date")
|
||||
date = headers.Get(xhttp.Date)
|
||||
}
|
||||
|
||||
// From the Amazon docs:
|
||||
@@ -393,8 +395,8 @@ func getStringToSignV2(method string, encodedResource, encodedQuery string, head
|
||||
// CanonicalizedResource;
|
||||
stringToSign := strings.Join([]string{
|
||||
method,
|
||||
headers.Get("Content-MD5"),
|
||||
headers.Get("Content-Type"),
|
||||
headers.Get(xhttp.ContentMD5),
|
||||
headers.Get(xhttp.ContentType),
|
||||
date,
|
||||
canonicalHeaders,
|
||||
}, "\n")
|
||||
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/sha256-simd"
|
||||
@@ -44,12 +45,12 @@ func skipContentSha256Cksum(r *http.Request) bool {
|
||||
)
|
||||
|
||||
if isRequestPresignedSignatureV4(r) {
|
||||
v, ok = r.URL.Query()["X-Amz-Content-Sha256"]
|
||||
v, ok = r.URL.Query()[xhttp.AmzContentSha256]
|
||||
if !ok {
|
||||
v, ok = r.Header["X-Amz-Content-Sha256"]
|
||||
v, ok = r.Header[xhttp.AmzContentSha256]
|
||||
}
|
||||
} else {
|
||||
v, ok = r.Header["X-Amz-Content-Sha256"]
|
||||
v, ok = r.Header[xhttp.AmzContentSha256]
|
||||
}
|
||||
|
||||
// If x-amz-content-sha256 is set and the value is not
|
||||
@@ -81,15 +82,15 @@ func getContentSha256Cksum(r *http.Request, stype serviceType) string {
|
||||
// X-Amz-Content-Sha256, if not set in presigned requests, checksum
|
||||
// will default to 'UNSIGNED-PAYLOAD'.
|
||||
defaultSha256Cksum = unsignedPayload
|
||||
v, ok = r.URL.Query()["X-Amz-Content-Sha256"]
|
||||
v, ok = r.URL.Query()[xhttp.AmzContentSha256]
|
||||
if !ok {
|
||||
v, ok = r.Header["X-Amz-Content-Sha256"]
|
||||
v, ok = r.Header[xhttp.AmzContentSha256]
|
||||
}
|
||||
} else {
|
||||
// X-Amz-Content-Sha256, if not set in signed requests, checksum
|
||||
// will default to sha256([]byte("")).
|
||||
defaultSha256Cksum = emptySHA256
|
||||
v, ok = r.Header["X-Amz-Content-Sha256"]
|
||||
v, ok = r.Header[xhttp.AmzContentSha256]
|
||||
}
|
||||
|
||||
// We found 'X-Amz-Content-Sha256' return the captured value.
|
||||
|
||||
+20
-19
@@ -36,6 +36,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v6/pkg/s3utils"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
sha256 "github.com/minio/sha256-simd"
|
||||
)
|
||||
|
||||
@@ -172,7 +173,7 @@ func doesPolicySignatureV4Match(formValues http.Header) APIErrorCode {
|
||||
region := globalServerConfig.GetRegion()
|
||||
|
||||
// Parse credential tag.
|
||||
credHeader, err := parseCredentialHeader("Credential="+formValues.Get("X-Amz-Credential"), region, serviceS3)
|
||||
credHeader, err := parseCredentialHeader("Credential="+formValues.Get(xhttp.AmzCredential), region, serviceS3)
|
||||
if err != ErrNone {
|
||||
return ErrMissingFields
|
||||
}
|
||||
@@ -189,7 +190,7 @@ func doesPolicySignatureV4Match(formValues http.Header) APIErrorCode {
|
||||
newSignature := getSignature(signingKey, formValues.Get("Policy"))
|
||||
|
||||
// Verify signature.
|
||||
if !compareSignatureV4(newSignature, formValues.Get("X-Amz-Signature")) {
|
||||
if !compareSignatureV4(newSignature, formValues.Get(xhttp.AmzSignature)) {
|
||||
return ErrSignatureDoesNotMatch
|
||||
}
|
||||
|
||||
@@ -223,11 +224,11 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
|
||||
|
||||
// Construct new query.
|
||||
query := make(url.Values)
|
||||
if req.URL.Query().Get("X-Amz-Content-Sha256") != "" {
|
||||
query.Set("X-Amz-Content-Sha256", hashedPayload)
|
||||
if req.URL.Query().Get(xhttp.AmzContentSha256) != "" {
|
||||
query.Set(xhttp.AmzContentSha256, hashedPayload)
|
||||
}
|
||||
|
||||
query.Set("X-Amz-Algorithm", signV4Algorithm)
|
||||
query.Set(xhttp.AmzAlgorithm, signV4Algorithm)
|
||||
|
||||
// If the host which signed the request is slightly ahead in time (by less than globalMaxSkewTime) the
|
||||
// request should still be allowed.
|
||||
@@ -244,10 +245,10 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
|
||||
expireSeconds := int(pSignValues.Expires / time.Second)
|
||||
|
||||
// Construct the query.
|
||||
query.Set("X-Amz-Date", t.Format(iso8601Format))
|
||||
query.Set("X-Amz-Expires", strconv.Itoa(expireSeconds))
|
||||
query.Set("X-Amz-SignedHeaders", getSignedHeaders(extractedSignedHeaders))
|
||||
query.Set("X-Amz-Credential", cred.AccessKey+"/"+pSignValues.Credential.getScope())
|
||||
query.Set(xhttp.AmzDate, t.Format(iso8601Format))
|
||||
query.Set(xhttp.AmzExpires, strconv.Itoa(expireSeconds))
|
||||
query.Set(xhttp.AmzSignedHeaders, getSignedHeaders(extractedSignedHeaders))
|
||||
query.Set(xhttp.AmzCredential, cred.AccessKey+"/"+pSignValues.Credential.getScope())
|
||||
|
||||
// Save other headers available in the request parameters.
|
||||
for k, v := range req.URL.Query() {
|
||||
@@ -273,24 +274,24 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
|
||||
encodedQuery := query.Encode()
|
||||
|
||||
// Verify if date query is same.
|
||||
if req.URL.Query().Get("X-Amz-Date") != query.Get("X-Amz-Date") {
|
||||
if req.URL.Query().Get(xhttp.AmzDate) != query.Get(xhttp.AmzDate) {
|
||||
return ErrSignatureDoesNotMatch
|
||||
}
|
||||
// Verify if expires query is same.
|
||||
if req.URL.Query().Get("X-Amz-Expires") != query.Get("X-Amz-Expires") {
|
||||
if req.URL.Query().Get(xhttp.AmzExpires) != query.Get(xhttp.AmzExpires) {
|
||||
return ErrSignatureDoesNotMatch
|
||||
}
|
||||
// Verify if signed headers query is same.
|
||||
if req.URL.Query().Get("X-Amz-SignedHeaders") != query.Get("X-Amz-SignedHeaders") {
|
||||
if req.URL.Query().Get(xhttp.AmzSignedHeaders) != query.Get(xhttp.AmzSignedHeaders) {
|
||||
return ErrSignatureDoesNotMatch
|
||||
}
|
||||
// Verify if credential query is same.
|
||||
if req.URL.Query().Get("X-Amz-Credential") != query.Get("X-Amz-Credential") {
|
||||
if req.URL.Query().Get(xhttp.AmzCredential) != query.Get(xhttp.AmzCredential) {
|
||||
return ErrSignatureDoesNotMatch
|
||||
}
|
||||
// Verify if sha256 payload query is same.
|
||||
if req.URL.Query().Get("X-Amz-Content-Sha256") != "" {
|
||||
if req.URL.Query().Get("X-Amz-Content-Sha256") != query.Get("X-Amz-Content-Sha256") {
|
||||
if req.URL.Query().Get(xhttp.AmzContentSha256) != "" {
|
||||
if req.URL.Query().Get(xhttp.AmzContentSha256) != query.Get(xhttp.AmzContentSha256) {
|
||||
return ErrContentSHA256Mismatch
|
||||
}
|
||||
}
|
||||
@@ -311,7 +312,7 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
|
||||
newSignature := getSignature(presignedSigningKey, presignedStringToSign)
|
||||
|
||||
// Verify signature.
|
||||
if !compareSignatureV4(req.URL.Query().Get("X-Amz-Signature"), newSignature) {
|
||||
if !compareSignatureV4(req.URL.Query().Get(xhttp.AmzSignature), newSignature) {
|
||||
return ErrSignatureDoesNotMatch
|
||||
}
|
||||
return ErrNone
|
||||
@@ -325,7 +326,7 @@ func doesSignatureMatch(hashedPayload string, r *http.Request, region string, st
|
||||
req := *r
|
||||
|
||||
// Save authorization header.
|
||||
v4Auth := req.Header.Get("Authorization")
|
||||
v4Auth := req.Header.Get(xhttp.Authorization)
|
||||
|
||||
// Parse signature version '4' header.
|
||||
signV4Values, err := parseSignV4(v4Auth, region, stype)
|
||||
@@ -346,8 +347,8 @@ func doesSignatureMatch(hashedPayload string, r *http.Request, region string, st
|
||||
|
||||
// Extract date, if not present throw error.
|
||||
var date string
|
||||
if date = req.Header.Get(http.CanonicalHeaderKey("x-amz-date")); date == "" {
|
||||
if date = r.Header.Get("Date"); date == "" {
|
||||
if date = req.Header.Get(xhttp.AmzDate); date == "" {
|
||||
if date = r.Header.Get(xhttp.Date); date == "" {
|
||||
return ErrMissingDateHeader
|
||||
}
|
||||
}
|
||||
|
||||
@@ -91,18 +91,17 @@ var errLessData = errors.New("less data available than what was requested")
|
||||
// errMoreData = returned when more data was sent by the caller than what it was supposed to.
|
||||
var errMoreData = errors.New("more data was sent than what was advertised")
|
||||
|
||||
// hashMisMatchError - represents a bit-rot hash verification failure
|
||||
// error.
|
||||
type hashMismatchError struct {
|
||||
expected string
|
||||
computed string
|
||||
// HashMismatchError represents a bit-rot hash verification failure error.
|
||||
type HashMismatchError struct {
|
||||
Expected string
|
||||
Computed string
|
||||
}
|
||||
|
||||
// error method for the hashMismatchError
|
||||
func (h hashMismatchError) Error() string {
|
||||
// Error method for the hashMismatchError
|
||||
func (h HashMismatchError) Error() string {
|
||||
return fmt.Sprintf(
|
||||
"Bitrot verification mismatch - expected %v, received %v",
|
||||
h.expected, h.computed)
|
||||
h.Expected, h.Computed)
|
||||
}
|
||||
|
||||
// Collection of basic errors.
|
||||
|
||||
@@ -52,6 +52,7 @@ type StorageAPI interface {
|
||||
StatFile(volume string, path string) (file FileInfo, err error)
|
||||
DeleteFile(volume string, path string) (err error)
|
||||
DeleteFileBulk(volume string, paths []string) (errs []error, err error)
|
||||
VerifyFile(volume, path string, algo BitrotAlgorithm, sum []byte, shardSize int64) error
|
||||
|
||||
// Write all data, syncs the data to disk.
|
||||
WriteAll(volume string, path string, reader io.Reader) (err error)
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"io"
|
||||
@@ -105,7 +106,7 @@ func toStorageErr(err error) error {
|
||||
fmt.Sscanf(err.Error(), "Bitrot verification mismatch - expected %s received %s", &expected, &received)
|
||||
// Go's Sscanf %s scans "," that comes after the expected hash, hence remove it. Providing "," in the format string does not help.
|
||||
expected = strings.TrimSuffix(expected, ",")
|
||||
bitrotErr := hashMismatchError{expected, received}
|
||||
bitrotErr := HashMismatchError{expected, received}
|
||||
return bitrotErr
|
||||
}
|
||||
return err
|
||||
@@ -433,6 +434,39 @@ func (client *storageRESTClient) getInstanceID() (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (client *storageRESTClient) VerifyFile(volume, path string, algo BitrotAlgorithm, sum []byte, shardSize int64) error {
|
||||
values := make(url.Values)
|
||||
values.Set(storageRESTVolume, volume)
|
||||
values.Set(storageRESTFilePath, path)
|
||||
values.Set(storageRESTBitrotAlgo, algo.String())
|
||||
values.Set(storageRESTLength, strconv.Itoa(int(shardSize)))
|
||||
if len(sum) != 0 {
|
||||
values.Set(storageRESTBitrotHash, hex.EncodeToString(sum))
|
||||
}
|
||||
respBody, err := client.call(storageRESTMethodVerifyFile, values, nil, -1)
|
||||
defer http.DrainBody(respBody)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reader := bufio.NewReader(respBody)
|
||||
for {
|
||||
b, err := reader.ReadByte()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if b != ' ' {
|
||||
reader.UnreadByte()
|
||||
break
|
||||
}
|
||||
}
|
||||
verifyResp := &VerifyFileResp{}
|
||||
err = gob.NewDecoder(reader).Decode(verifyResp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return toStorageErr(verifyResp.Err)
|
||||
}
|
||||
|
||||
// Close - marks the client as closed.
|
||||
func (client *storageRESTClient) Close() error {
|
||||
client.connected = false
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
|
||||
package cmd
|
||||
|
||||
const storageRESTVersion = "v6"
|
||||
const storageRESTVersion = "v7"
|
||||
const storageRESTPath = minioReservedBucketPath + "/storage/" + storageRESTVersion + "/"
|
||||
|
||||
const (
|
||||
@@ -38,6 +38,7 @@ const (
|
||||
storageRESTMethodDeleteFile = "deletefile"
|
||||
storageRESTMethodDeleteFileBulk = "deletefilebulk"
|
||||
storageRESTMethodRenameFile = "renamefile"
|
||||
storageRESTMethodVerifyFile = "verifyfile"
|
||||
storageRESTMethodGetInstanceID = "getinstanceid"
|
||||
)
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
)
|
||||
|
||||
@@ -114,7 +115,7 @@ func (s *storageRESTServer) GetInstanceID(w http.ResponseWriter, r *http.Request
|
||||
s.writeErrorResponse(w, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(s.instanceID)))
|
||||
w.Header().Set(xhttp.ContentLength, strconv.Itoa(len(s.instanceID)))
|
||||
w.Write([]byte(s.instanceID))
|
||||
w.(http.Flusher).Flush()
|
||||
}
|
||||
@@ -283,7 +284,7 @@ func (s *storageRESTServer) ReadAllHandler(w http.ResponseWriter, r *http.Reques
|
||||
s.writeErrorResponse(w, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(buf)))
|
||||
w.Header().Set(xhttp.ContentLength, strconv.Itoa(len(buf)))
|
||||
w.Write(buf)
|
||||
w.(http.Flusher).Flush()
|
||||
}
|
||||
@@ -327,7 +328,7 @@ func (s *storageRESTServer) ReadFileHandler(w http.ResponseWriter, r *http.Reque
|
||||
s.writeErrorResponse(w, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(buf)))
|
||||
w.Header().Set(xhttp.ContentLength, strconv.Itoa(len(buf)))
|
||||
w.Write(buf)
|
||||
w.(http.Flusher).Flush()
|
||||
}
|
||||
@@ -357,7 +358,7 @@ func (s *storageRESTServer) ReadFileStreamHandler(w http.ResponseWriter, r *http
|
||||
return
|
||||
}
|
||||
defer rc.Close()
|
||||
w.Header().Set("Content-Length", strconv.Itoa(length))
|
||||
w.Header().Set(xhttp.ContentLength, strconv.Itoa(length))
|
||||
|
||||
io.Copy(w, rc)
|
||||
w.(http.Flusher).Flush()
|
||||
@@ -486,6 +487,65 @@ func (s *storageRESTServer) RenameFileHandler(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
}
|
||||
|
||||
// Send whitespace to the client to avoid timeouts as bitrot verification can take time on spinning/slow disks.
|
||||
func sendWhiteSpaceVerifyFile(w http.ResponseWriter) <-chan struct{} {
|
||||
doneCh := make(chan struct{})
|
||||
go func() {
|
||||
ticker := time.NewTicker(time.Second * 10)
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
w.Write([]byte(" "))
|
||||
w.(http.Flusher).Flush()
|
||||
case doneCh <- struct{}{}:
|
||||
ticker.Stop()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
}()
|
||||
return doneCh
|
||||
}
|
||||
|
||||
// VerifyFileResp - VerifyFile()'s response.
|
||||
type VerifyFileResp struct {
|
||||
Err error
|
||||
}
|
||||
|
||||
// VerifyFile - Verify the file for bitrot errors.
|
||||
func (s *storageRESTServer) VerifyFile(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.IsValid(w, r) {
|
||||
return
|
||||
}
|
||||
vars := mux.Vars(r)
|
||||
volume := vars[storageRESTVolume]
|
||||
filePath := vars[storageRESTFilePath]
|
||||
shardSize, err := strconv.Atoi(vars[storageRESTLength])
|
||||
if err != nil {
|
||||
s.writeErrorResponse(w, err)
|
||||
return
|
||||
}
|
||||
hashStr := vars[storageRESTBitrotHash]
|
||||
var hash []byte
|
||||
if hashStr != "" {
|
||||
hash, err = hex.DecodeString(hashStr)
|
||||
if err != nil {
|
||||
s.writeErrorResponse(w, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
algoStr := vars[storageRESTBitrotAlgo]
|
||||
if algoStr == "" {
|
||||
s.writeErrorResponse(w, errInvalidArgument)
|
||||
return
|
||||
}
|
||||
algo := BitrotAlgorithmFromString(algoStr)
|
||||
doneCh := sendWhiteSpaceVerifyFile(w)
|
||||
err = s.storage.VerifyFile(volume, filePath, algo, hash, int64(shardSize))
|
||||
<-doneCh
|
||||
gob.NewEncoder(w).Encode(VerifyFileResp{err})
|
||||
}
|
||||
|
||||
// registerStorageRPCRouter - register storage rpc router.
|
||||
func registerStorageRESTHandlers(router *mux.Router, endpoints EndpointList) {
|
||||
for _, endpoint := range endpoints {
|
||||
@@ -533,6 +593,8 @@ func registerStorageRESTHandlers(router *mux.Router, endpoints EndpointList) {
|
||||
|
||||
subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodRenameFile).HandlerFunc(httpTraceHdrs(server.RenameFileHandler)).
|
||||
Queries(restQueries(storageRESTSrcVolume, storageRESTSrcPath, storageRESTDstVolume, storageRESTDstPath)...)
|
||||
subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodVerifyFile).HandlerFunc(httpTraceHdrs(server.VerifyFile)).
|
||||
Queries(restQueries(storageRESTVolume, storageRESTFilePath, storageRESTBitrotAlgo, storageRESTLength)...)
|
||||
subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodGetInstanceID).HandlerFunc(httpTraceAll(server.GetInstanceID))
|
||||
}
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ import (
|
||||
"time"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
sha256 "github.com/minio/sha256-simd"
|
||||
)
|
||||
@@ -69,7 +70,7 @@ func calculateSeedSignature(r *http.Request) (cred auth.Credentials, signature s
|
||||
req := *r
|
||||
|
||||
// Save authorization header.
|
||||
v4Auth := req.Header.Get("Authorization")
|
||||
v4Auth := req.Header.Get(xhttp.Authorization)
|
||||
|
||||
// Parse signature version '4' header.
|
||||
signV4Values, errCode := parseSignV4(v4Auth, globalServerConfig.GetRegion(), serviceS3)
|
||||
@@ -81,7 +82,7 @@ func calculateSeedSignature(r *http.Request) (cred auth.Credentials, signature s
|
||||
payload := streamingContentSHA256
|
||||
|
||||
// Payload for STREAMING signature should be 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'
|
||||
if payload != req.Header.Get("X-Amz-Content-Sha256") {
|
||||
if payload != req.Header.Get(xhttp.AmzContentSha256) {
|
||||
return cred, "", "", time.Time{}, ErrContentSHA256Mismatch
|
||||
}
|
||||
|
||||
|
||||
+3
-1
@@ -19,12 +19,14 @@ package cmd
|
||||
import (
|
||||
"encoding/xml"
|
||||
"net/http"
|
||||
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
)
|
||||
|
||||
// writeSTSErrorRespone writes error headers
|
||||
func writeSTSErrorResponse(w http.ResponseWriter, err STSError) {
|
||||
// Generate error response.
|
||||
stsErrorResponse := getSTSErrorResponse(err, w.Header().Get(responseRequestIDKey))
|
||||
stsErrorResponse := getSTSErrorResponse(err, w.Header().Get(xhttp.AmzRequestID))
|
||||
encodedErrorResponse := encodeResponse(stsErrorResponse)
|
||||
writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeXML)
|
||||
}
|
||||
|
||||
+67
-19
@@ -17,13 +17,17 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
iampolicy "github.com/minio/minio/pkg/iam/policy"
|
||||
"github.com/minio/minio/pkg/iam/validator"
|
||||
"github.com/minio/minio/pkg/wildcard"
|
||||
)
|
||||
@@ -50,16 +54,16 @@ func registerSTSRouter(router *mux.Router) {
|
||||
stsRouter := router.NewRoute().PathPrefix("/").Subrouter()
|
||||
|
||||
// Assume roles with no JWT, handles AssumeRole.
|
||||
stsRouter.Methods("POST").MatcherFunc(func(r *http.Request, rm *mux.RouteMatch) bool {
|
||||
ctypeOk := wildcard.MatchSimple("application/x-www-form-urlencoded*", r.Header.Get("Content-Type"))
|
||||
authOk := wildcard.MatchSimple("AWS4-HMAC-SHA256*", r.Header.Get("Authorization"))
|
||||
stsRouter.Methods(http.MethodPost).MatcherFunc(func(r *http.Request, rm *mux.RouteMatch) bool {
|
||||
ctypeOk := wildcard.MatchSimple("application/x-www-form-urlencoded*", r.Header.Get(xhttp.ContentType))
|
||||
authOk := wildcard.MatchSimple(signV4Algorithm+"*", r.Header.Get(xhttp.Authorization))
|
||||
noQueries := len(r.URL.Query()) == 0
|
||||
return ctypeOk && authOk && noQueries
|
||||
}).HandlerFunc(httpTraceAll(sts.AssumeRole))
|
||||
|
||||
// Assume roles with JWT handler, handles both ClientGrants and WebIdentity.
|
||||
stsRouter.Methods("POST").MatcherFunc(func(r *http.Request, rm *mux.RouteMatch) bool {
|
||||
ctypeOk := wildcard.MatchSimple("application/x-www-form-urlencoded*", r.Header.Get("Content-Type"))
|
||||
ctypeOk := wildcard.MatchSimple("application/x-www-form-urlencoded*", r.Header.Get(xhttp.ContentType))
|
||||
noQueries := len(r.URL.Query()) == 0
|
||||
return ctypeOk && noQueries
|
||||
}).HandlerFunc(httpTraceAll(sts.AssumeRoleWithJWT))
|
||||
@@ -124,11 +128,6 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if r.Form.Get("Policy") != "" {
|
||||
writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
|
||||
return
|
||||
}
|
||||
|
||||
if r.Form.Get("Version") != stsAPIVersion {
|
||||
logger.LogIf(ctx, fmt.Errorf("Invalid STS API version %s, expecting %s", r.Form.Get("Version"), stsAPIVersion))
|
||||
writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSMissingParameter))
|
||||
@@ -147,6 +146,29 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
|
||||
ctx = newContext(r, w, action)
|
||||
defer logger.AuditLog(w, r, action, nil)
|
||||
|
||||
sessionPolicyStr := r.Form.Get("Policy")
|
||||
// https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html
|
||||
// The plain text that you use for both inline and managed session
|
||||
// policies shouldn't exceed 2048 characters.
|
||||
if len(sessionPolicyStr) > 2048 {
|
||||
writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
|
||||
return
|
||||
}
|
||||
|
||||
if len(sessionPolicyStr) > 0 {
|
||||
sessionPolicy, err := iampolicy.ParseConfig(bytes.NewReader([]byte(sessionPolicyStr)))
|
||||
if err != nil {
|
||||
writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
|
||||
return
|
||||
}
|
||||
|
||||
// Version in policy must not be empty
|
||||
if sessionPolicy.Version == "" {
|
||||
writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
var err error
|
||||
m := make(map[string]interface{})
|
||||
m["exp"], err = validator.GetDefaultExpiration(r.Form.Get("DurationSeconds"))
|
||||
@@ -171,7 +193,11 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
|
||||
// This policy is the policy associated with the user
|
||||
// requesting for temporary credentials. The temporary
|
||||
// credentials will inherit the same policy requirements.
|
||||
m["policy"] = policyName
|
||||
m[iampolicy.PolicyName] = policyName
|
||||
|
||||
if len(sessionPolicyStr) > 0 {
|
||||
m[iampolicy.SessionPolicyName] = base64.StdEncoding.EncodeToString([]byte(sessionPolicyStr))
|
||||
}
|
||||
|
||||
secret := globalServerConfig.GetCredential().SecretKey
|
||||
cred, err := auth.GetNewCredentialsWithMetadata(m, secret)
|
||||
@@ -202,7 +228,7 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
|
||||
},
|
||||
}
|
||||
|
||||
assumeRoleResponse.ResponseMetadata.RequestID = w.Header().Get(responseRequestIDKey)
|
||||
assumeRoleResponse.ResponseMetadata.RequestID = w.Header().Get(xhttp.AmzRequestID)
|
||||
writeSuccessResponseXML(w, encodeResponse(assumeRoleResponse))
|
||||
}
|
||||
|
||||
@@ -216,11 +242,6 @@ func (sts *stsAPIHandlers) AssumeRoleWithJWT(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
|
||||
if r.Form.Get("Policy") != "" {
|
||||
writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
|
||||
return
|
||||
}
|
||||
|
||||
if r.Form.Get("Version") != stsAPIVersion {
|
||||
logger.LogIf(ctx, fmt.Errorf("Invalid STS API version %s, expecting %s", r.Form.Get("Version"), stsAPIVersion))
|
||||
writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSMissingParameter))
|
||||
@@ -276,6 +297,29 @@ func (sts *stsAPIHandlers) AssumeRoleWithJWT(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
|
||||
sessionPolicyStr := r.Form.Get("Policy")
|
||||
// https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html
|
||||
// The plain text that you use for both inline and managed session
|
||||
// policies shouldn't exceed 2048 characters.
|
||||
if len(sessionPolicyStr) > 2048 {
|
||||
writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
|
||||
return
|
||||
}
|
||||
|
||||
if len(sessionPolicyStr) > 0 {
|
||||
sessionPolicy, err := iampolicy.ParseConfig(bytes.NewReader([]byte(sessionPolicyStr)))
|
||||
if err != nil {
|
||||
writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
|
||||
return
|
||||
}
|
||||
|
||||
// Version in policy must not be empty
|
||||
if sessionPolicy.Version == "" {
|
||||
writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
secret := globalServerConfig.GetCredential().SecretKey
|
||||
cred, err := auth.GetNewCredentialsWithMetadata(m, secret)
|
||||
if err != nil {
|
||||
@@ -289,7 +333,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithJWT(w http.ResponseWriter, r *http.Requ
|
||||
// be set and configured on your identity provider as part of
|
||||
// JWT custom claims.
|
||||
var policyName string
|
||||
if v, ok := m["policy"]; ok {
|
||||
if v, ok := m[iampolicy.PolicyName]; ok {
|
||||
policyName, _ = v.(string)
|
||||
}
|
||||
|
||||
@@ -298,6 +342,10 @@ func (sts *stsAPIHandlers) AssumeRoleWithJWT(w http.ResponseWriter, r *http.Requ
|
||||
subFromToken, _ = v.(string)
|
||||
}
|
||||
|
||||
if len(sessionPolicyStr) > 0 {
|
||||
m[iampolicy.SessionPolicyName] = base64.StdEncoding.EncodeToString([]byte(sessionPolicyStr))
|
||||
}
|
||||
|
||||
// Set the newly generated credentials.
|
||||
if err = globalIAMSys.SetTempUser(cred.AccessKey, cred, policyName); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
@@ -322,7 +370,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithJWT(w http.ResponseWriter, r *http.Requ
|
||||
SubjectFromToken: subFromToken,
|
||||
},
|
||||
}
|
||||
clientGrantsResponse.ResponseMetadata.RequestID = w.Header().Get(responseRequestIDKey)
|
||||
clientGrantsResponse.ResponseMetadata.RequestID = w.Header().Get(xhttp.AmzRequestID)
|
||||
encodedSuccessResponse = encodeResponse(clientGrantsResponse)
|
||||
case webIdentity:
|
||||
webIdentityResponse := &AssumeRoleWithWebIdentityResponse{
|
||||
@@ -331,7 +379,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithJWT(w http.ResponseWriter, r *http.Requ
|
||||
SubjectFromWebIdentityToken: subFromToken,
|
||||
},
|
||||
}
|
||||
webIdentityResponse.ResponseMetadata.RequestID = w.Header().Get(responseRequestIDKey)
|
||||
webIdentityResponse.ResponseMetadata.RequestID = w.Header().Get(xhttp.AmzRequestID)
|
||||
encodedSuccessResponse = encodeResponse(webIdentityResponse)
|
||||
}
|
||||
|
||||
|
||||
@@ -2135,7 +2135,7 @@ func registerBucketLevelFunc(bucket *mux.Router, api objectAPIHandlers, apiFunct
|
||||
func registerAPIFunctions(muxRouter *mux.Router, objLayer ObjectLayer, apiFunctions ...string) {
|
||||
if len(apiFunctions) == 0 {
|
||||
// Register all api endpoints by default.
|
||||
registerAPIRouter(muxRouter, true)
|
||||
registerAPIRouter(muxRouter, true, false)
|
||||
return
|
||||
}
|
||||
// API Router.
|
||||
@@ -2176,7 +2176,7 @@ func initTestAPIEndPoints(objLayer ObjectLayer, apiFunctions []string) http.Hand
|
||||
registerAPIFunctions(muxRouter, objLayer, apiFunctions...)
|
||||
return muxRouter
|
||||
}
|
||||
registerAPIRouter(muxRouter, true)
|
||||
registerAPIRouter(muxRouter, true, false)
|
||||
return muxRouter
|
||||
}
|
||||
|
||||
|
||||
+18
-3
@@ -30,12 +30,12 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/fatih/color"
|
||||
"github.com/inconshreveable/go-update"
|
||||
"github.com/minio/cli"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
_ "github.com/minio/sha256-simd" // Needed for sha256 hash verifier.
|
||||
"github.com/segmentio/go-prompt"
|
||||
)
|
||||
|
||||
// Check for new software updates.
|
||||
@@ -452,7 +452,7 @@ func getUpdateInfo(timeout time.Duration, mode string) (updateMsg string, sha256
|
||||
|
||||
func doUpdate(sha256Hex string, latestReleaseTime time.Time, ok bool) (updateStatusMsg string, err error) {
|
||||
if !ok {
|
||||
updateStatusMsg = colorGreenBold("MinIO update to version RELEASE.%s canceled.",
|
||||
updateStatusMsg = colorRedBold("MinIO update to version RELEASE.%s canceled.",
|
||||
latestReleaseTime.Format(minioReleaseTagTimeLayout))
|
||||
return updateStatusMsg, nil
|
||||
}
|
||||
@@ -482,10 +482,25 @@ func doUpdate(sha256Hex string, latestReleaseTime time.Time, ok bool) (updateSta
|
||||
latestReleaseTime.Format(minioReleaseTagTimeLayout)), nil
|
||||
}
|
||||
|
||||
// Confirm continues prompting until the input is boolean-ish.
|
||||
func confirm(prompt string, args ...interface{}) bool {
|
||||
for {
|
||||
var s string
|
||||
fmt.Fprintf(color.Output, prompt+": ", args...)
|
||||
fmt.Scanln(&s)
|
||||
switch s {
|
||||
case "Yes", "yes", "y", "Y":
|
||||
return true
|
||||
case "No", "no", "n", "N":
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func shouldUpdate(quiet bool, sha256Hex string, latestReleaseTime time.Time) (ok bool) {
|
||||
ok = true
|
||||
if !quiet {
|
||||
ok = prompt.Confirm(colorGreenBold("Update to RELEASE.%s [%s]", latestReleaseTime.Format(minioReleaseTagTimeLayout), "yes"))
|
||||
ok = confirm(colorGreenBold("Update to RELEASE.%s ? [%s]", latestReleaseTime.Format(minioReleaseTagTimeLayout), "y/n"))
|
||||
}
|
||||
return ok
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user