Compare commits
30 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1cfd4a48d9 | |||
| 993a79d9c6 | |||
| 0c16b1c9a7 | |||
| 900cc27b51 | |||
| 8528017ad3 | |||
| a73da7755e | |||
| 2c0b3cadfc | |||
| 763fce909b | |||
| da8214845a | |||
| 74e2fe0879 | |||
| cb7f9ba286 | |||
| 7906f464fd | |||
| 7e4c9a9e1e | |||
| f6fd407e47 | |||
| 46ced81f41 | |||
| 7a02faab72 | |||
| e252114f06 | |||
| 39b3e4f9b3 | |||
| 158b8c2e86 | |||
| c4c79f61ce | |||
| 59e1d94770 | |||
| 59e847aebe | |||
| a13b58f630 | |||
| 9e7a19d6b9 | |||
| 16c648b109 | |||
| c871456269 | |||
| 55aa20595f | |||
| 707ed2b302 | |||
| 78be3f8947 | |||
| 67d508b214 |
@@ -1,4 +1,4 @@
|
||||
# MinIO Contribution Guide [](https://slack.min.io) [](https://goreportcard.com/report/minio/minio) [](https://hub.docker.com/r/minio/minio/) [](https://codecov.io/gh/minio/minio)
|
||||
# MinIO Contribution Guide [](https://slack.min.io) [](https://goreportcard.com/report/minio/minio) [](https://hub.docker.com/r/minio/minio/)
|
||||
|
||||
``MinIO`` community welcomes your contribution. To make the process as seamless as possible, we recommend you read this contribution guide.
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# MinIO Quickstart Guide
|
||||
[](https://slack.min.io) [](https://goreportcard.com/report/minio/minio) [](https://hub.docker.com/r/minio/minio/) [](https://codecov.io/gh/minio/minio)
|
||||
[](https://slack.min.io) [](https://goreportcard.com/report/minio/minio) [](https://hub.docker.com/r/minio/minio/)
|
||||
|
||||
MinIO is an object storage server released under Apache License v2.0. It is compatible with Amazon S3 cloud storage service. It is best suited for storing unstructured data such as photos, videos, log files, backups and container / VM images. Size of an object can range from a few KBs to a maximum of 5TB.
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# MinIO Quickstart Guide [](https://slack.min.io) [](https://goreportcard.com/report/minio/minio) [](https://hub.docker.com/r/minio/minio/) [](https://codecov.io/gh/minio/minio)
|
||||
# MinIO Quickstart Guide [](https://slack.min.io) [](https://goreportcard.com/report/minio/minio) [](https://hub.docker.com/r/minio/minio/)
|
||||
|
||||
MinIO 是一个基于Apache License v2.0开源协议的对象存储服务。它兼容亚马逊S3云存储服务接口,非常适合于存储大容量非结构化的数据,例如图片、视频、日志文件、备份数据和容器/虚拟机镜像等,而一个对象文件可以是任意大小,从几kb到最大5T不等。
|
||||
|
||||
|
||||
@@ -18,157 +18,237 @@ import React from "react"
|
||||
import { connect } from "react-redux"
|
||||
import web from "../web"
|
||||
import * as alertActions from "../alert/actions"
|
||||
import { getRandomAccessKey, getRandomSecretKey } from "../utils"
|
||||
import jwtDecode from "jwt-decode"
|
||||
import classNames from "classnames"
|
||||
|
||||
import {
|
||||
Tooltip,
|
||||
Modal,
|
||||
ModalBody,
|
||||
ModalHeader,
|
||||
OverlayTrigger
|
||||
} from "react-bootstrap"
|
||||
import { Modal, ModalBody, ModalHeader } from "react-bootstrap"
|
||||
import InputGroup from "./InputGroup"
|
||||
import { ACCESS_KEY_MIN_LENGTH, SECRET_KEY_MIN_LENGTH } from "../constants"
|
||||
|
||||
export class ChangePasswordModal extends React.Component {
|
||||
constructor(props) {
|
||||
super(props)
|
||||
this.state = {
|
||||
accessKey: "",
|
||||
secretKey: "",
|
||||
keysReadOnly: false
|
||||
currentAccessKey: "",
|
||||
currentSecretKey: "",
|
||||
currentSecretKeyVisible: false,
|
||||
newAccessKey: "",
|
||||
newSecretKey: "",
|
||||
newSecretKeyVisible: false
|
||||
}
|
||||
}
|
||||
// When its shown, it loads the access key and secret key.
|
||||
// When its shown, it loads the access key from JWT token
|
||||
componentWillMount() {
|
||||
const { serverInfo } = this.props
|
||||
|
||||
// Check environment variables first.
|
||||
if (serverInfo.info.isEnvCreds || serverInfo.info.isWorm) {
|
||||
this.setState({
|
||||
accessKey: "xxxxxxxxx",
|
||||
secretKey: "xxxxxxxxx",
|
||||
keysReadOnly: true
|
||||
})
|
||||
} else {
|
||||
web.GetAuth().then(data => {
|
||||
this.setState({
|
||||
accessKey: data.accessKey,
|
||||
secretKey: data.secretKey
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Handle field changes from inside the modal.
|
||||
accessKeyChange(e) {
|
||||
const token = jwtDecode(web.GetToken())
|
||||
this.setState({
|
||||
accessKey: e.target.value
|
||||
})
|
||||
}
|
||||
|
||||
secretKeyChange(e) {
|
||||
this.setState({
|
||||
secretKey: e.target.value
|
||||
})
|
||||
}
|
||||
|
||||
secretKeyVisible(secretKeyVisible) {
|
||||
this.setState({
|
||||
secretKeyVisible
|
||||
currentAccessKey: token.sub,
|
||||
newAccessKey: token.sub
|
||||
})
|
||||
}
|
||||
|
||||
// Save the auth params and set them.
|
||||
setAuth(e) {
|
||||
const { showAlert } = this.props
|
||||
const accessKey = this.state.accessKey
|
||||
const secretKey = this.state.secretKey
|
||||
web
|
||||
.SetAuth({
|
||||
accessKey,
|
||||
secretKey
|
||||
})
|
||||
.then(data => {
|
||||
showAlert({
|
||||
type: "success",
|
||||
message: "Changed credentials"
|
||||
|
||||
if (this.canUpdateCredentials()) {
|
||||
const currentAccessKey = this.state.currentAccessKey
|
||||
const currentSecretKey = this.state.currentSecretKey
|
||||
const newAccessKey = this.state.newAccessKey
|
||||
const newSecretKey = this.state.newSecretKey
|
||||
web
|
||||
.SetAuth({
|
||||
currentAccessKey,
|
||||
currentSecretKey,
|
||||
newAccessKey,
|
||||
newSecretKey
|
||||
})
|
||||
})
|
||||
.catch(err => {
|
||||
showAlert({
|
||||
type: "danger",
|
||||
message: err.message
|
||||
.then(data => {
|
||||
showAlert({
|
||||
type: "success",
|
||||
message: "Credentials updated successfully."
|
||||
})
|
||||
})
|
||||
})
|
||||
.catch(err => {
|
||||
showAlert({
|
||||
type: "danger",
|
||||
message: err.message
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
generateAuth(e) {
|
||||
web.GenerateAuth().then(data => {
|
||||
const { serverInfo } = this.props
|
||||
// Generate random access key only for root user
|
||||
if (!serverInfo.userInfo.isIAMUser) {
|
||||
this.setState({
|
||||
accessKey: data.accessKey,
|
||||
secretKey: data.secretKey,
|
||||
secretKeyVisible: true
|
||||
newAccessKey: getRandomAccessKey()
|
||||
})
|
||||
}
|
||||
|
||||
this.setState({
|
||||
newSecretKey: getRandomSecretKey(),
|
||||
newSecretKeyVisible: true
|
||||
})
|
||||
}
|
||||
|
||||
canChangePassword() {
|
||||
const { serverInfo } = this.props
|
||||
// Password change is not allowed in WORM mode
|
||||
if (serverInfo.info.isWorm) {
|
||||
return false
|
||||
}
|
||||
|
||||
// When credentials are set on ENV, password change not allowed for owner
|
||||
if (serverInfo.info.isEnvCreds && !serverInfo.userInfo.isIAMUser) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
canUpdateCredentials() {
|
||||
return (
|
||||
this.state.currentAccessKey.length > 0 &&
|
||||
this.state.currentSecretKey.length > 0 &&
|
||||
this.state.newAccessKey.length >= ACCESS_KEY_MIN_LENGTH &&
|
||||
this.state.newSecretKey.length >= SECRET_KEY_MIN_LENGTH
|
||||
)
|
||||
}
|
||||
|
||||
render() {
|
||||
const { hideChangePassword } = this.props
|
||||
const { hideChangePassword, serverInfo } = this.props
|
||||
const allowChangePassword = this.canChangePassword()
|
||||
|
||||
if (!allowChangePassword) {
|
||||
return (
|
||||
<Modal bsSize="sm" animation={false} show={true}>
|
||||
<ModalHeader>Change Password</ModalHeader>
|
||||
<ModalBody>
|
||||
Credentials of this user cannot be updated through MinIO Browser.
|
||||
</ModalBody>
|
||||
<div className="modal-footer">
|
||||
<button
|
||||
id="cancel-change-password"
|
||||
className="btn btn-link"
|
||||
onClick={hideChangePassword}
|
||||
>
|
||||
Close
|
||||
</button>
|
||||
</div>
|
||||
</Modal>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<Modal bsSize="sm" animation={false} show={true}>
|
||||
<ModalHeader>Change Password</ModalHeader>
|
||||
<ModalBody className="m-t-20">
|
||||
<InputGroup
|
||||
value={this.state.accessKey}
|
||||
onChange={this.accessKeyChange.bind(this)}
|
||||
id="accessKey"
|
||||
label="Access Key"
|
||||
name="accesskey"
|
||||
type="text"
|
||||
spellCheck="false"
|
||||
required="required"
|
||||
autoComplete="false"
|
||||
align="ig-left"
|
||||
readonly={this.state.keysReadOnly}
|
||||
/>
|
||||
<i
|
||||
onClick={this.secretKeyVisible.bind(
|
||||
this,
|
||||
!this.state.secretKeyVisible
|
||||
<div className="has-toggle-password">
|
||||
<InputGroup
|
||||
value={this.state.currentAccessKey}
|
||||
id="currentAccessKey"
|
||||
label="Current Access Key"
|
||||
name="currentAccesskey"
|
||||
type="text"
|
||||
spellCheck="false"
|
||||
required="required"
|
||||
autoComplete="false"
|
||||
align="ig-left"
|
||||
readonly={true}
|
||||
/>
|
||||
|
||||
<i
|
||||
onClick={() => {
|
||||
this.setState({
|
||||
currentSecretKeyVisible: !this.state.currentSecretKeyVisible
|
||||
})
|
||||
}}
|
||||
className={
|
||||
"toggle-password fa fa-eye " +
|
||||
(this.state.currentSecretKeyVisible ? "toggled" : "")
|
||||
}
|
||||
/>
|
||||
<InputGroup
|
||||
value={this.state.currentSecretKey}
|
||||
onChange={e => {
|
||||
this.setState({ currentSecretKey: e.target.value })
|
||||
}}
|
||||
id="currentSecretKey"
|
||||
label="Current Secret Key"
|
||||
name="currentSecretKey"
|
||||
type={this.state.currentSecretKeyVisible ? "text" : "password"}
|
||||
spellCheck="false"
|
||||
required="required"
|
||||
autoComplete="false"
|
||||
align="ig-left"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="has-toggle-password m-t-30">
|
||||
{!serverInfo.userInfo.isIAMUser && (
|
||||
<InputGroup
|
||||
value={this.state.newAccessKey}
|
||||
id="newAccessKey"
|
||||
label={"New Access Key"}
|
||||
name="newAccesskey"
|
||||
type="text"
|
||||
spellCheck="false"
|
||||
required="required"
|
||||
autoComplete="false"
|
||||
align="ig-left"
|
||||
onChange={e => {
|
||||
this.setState({ newAccessKey: e.target.value })
|
||||
}}
|
||||
readonly={serverInfo.userInfo.isIAMUser}
|
||||
/>
|
||||
)}
|
||||
className={
|
||||
"toggle-password fa fa-eye " +
|
||||
(this.state.secretKeyVisible ? "toggled" : "")
|
||||
}
|
||||
/>
|
||||
<InputGroup
|
||||
value={this.state.secretKey}
|
||||
onChange={this.secretKeyChange.bind(this)}
|
||||
id="secretKey"
|
||||
label="Secret Key"
|
||||
name="accesskey"
|
||||
type={this.state.secretKeyVisible ? "text" : "password"}
|
||||
spellCheck="false"
|
||||
required="required"
|
||||
autoComplete="false"
|
||||
align="ig-left"
|
||||
readonly={this.state.keysReadOnly}
|
||||
/>
|
||||
|
||||
<i
|
||||
onClick={() => {
|
||||
this.setState({
|
||||
newSecretKeyVisible: !this.state.newSecretKeyVisible
|
||||
})
|
||||
}}
|
||||
className={
|
||||
"toggle-password fa fa-eye " +
|
||||
(this.state.newSecretKeyVisible ? "toggled" : "")
|
||||
}
|
||||
/>
|
||||
<InputGroup
|
||||
value={this.state.newSecretKey}
|
||||
onChange={e => {
|
||||
this.setState({ newSecretKey: e.target.value })
|
||||
}}
|
||||
id="newSecretKey"
|
||||
label="New Secret Key"
|
||||
name="newSecretKey"
|
||||
type={this.state.newSecretKeyVisible ? "text" : "password"}
|
||||
spellCheck="false"
|
||||
required="required"
|
||||
autoComplete="false"
|
||||
align="ig-left"
|
||||
onChange={e => {
|
||||
this.setState({ newSecretKey: e.target.value })
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</ModalBody>
|
||||
<div className="modal-footer">
|
||||
<button
|
||||
id="generate-keys"
|
||||
className={
|
||||
"btn btn-primary " + (this.state.keysReadOnly ? "hidden" : "")
|
||||
}
|
||||
className={"btn btn-primary"}
|
||||
onClick={this.generateAuth.bind(this)}
|
||||
>
|
||||
Generate
|
||||
</button>
|
||||
<button
|
||||
id="update-keys"
|
||||
className={
|
||||
"btn btn-success " + (this.state.keysReadOnly ? "hidden" : "")
|
||||
}
|
||||
className={classNames({
|
||||
btn: true,
|
||||
"btn-success": this.canUpdateCredentials()
|
||||
})}
|
||||
disabled={!this.canUpdateCredentials()}
|
||||
onClick={this.setAuth.bind(this)}
|
||||
>
|
||||
Update
|
||||
@@ -198,4 +278,7 @@ const mapDispatchToProps = dispatch => {
|
||||
}
|
||||
}
|
||||
|
||||
export default connect(mapStateToProps, mapDispatchToProps)(ChangePasswordModal)
|
||||
export default connect(
|
||||
mapStateToProps,
|
||||
mapDispatchToProps
|
||||
)(ChangePasswordModal)
|
||||
|
||||
@@ -17,21 +17,38 @@
|
||||
import React from "react"
|
||||
import { shallow, mount } from "enzyme"
|
||||
import { ChangePasswordModal } from "../ChangePasswordModal"
|
||||
import jwtDecode from "jwt-decode"
|
||||
|
||||
jest.mock("jwt-decode")
|
||||
|
||||
jwtDecode.mockImplementation(() => ({ sub: "minio" }))
|
||||
|
||||
jest.mock("../../web", () => ({
|
||||
GetAuth: jest.fn(() => {
|
||||
return Promise.resolve({ accessKey: "test1", secretKey: "test2" })
|
||||
}),
|
||||
GenerateAuth: jest.fn(() => {
|
||||
return Promise.resolve({ accessKey: "gen1", secretKey: "gen2" })
|
||||
}),
|
||||
SetAuth: jest.fn(({ accessKey, secretKey }) => {
|
||||
if (accessKey == "test3" && secretKey == "test4") {
|
||||
return Promise.resolve({})
|
||||
} else {
|
||||
return Promise.reject({ message: "Error" })
|
||||
SetAuth: jest.fn(
|
||||
({ currentAccessKey, currentSecretKey, newAccessKey, newSecretKey }) => {
|
||||
if (
|
||||
currentAccessKey == "minio" &&
|
||||
currentSecretKey == "minio123" &&
|
||||
newAccessKey == "test" &&
|
||||
newSecretKey == "test1234"
|
||||
) {
|
||||
return Promise.resolve({})
|
||||
} else {
|
||||
return Promise.reject({
|
||||
message: "Error"
|
||||
})
|
||||
}
|
||||
}
|
||||
})
|
||||
),
|
||||
GetToken: jest.fn(() => "")
|
||||
}))
|
||||
|
||||
jest.mock("../../utils", () => ({
|
||||
getRandomAccessKey: () => "raccesskey",
|
||||
getRandomSecretKey: () => "rsecretkey"
|
||||
}))
|
||||
|
||||
describe("ChangePasswordModal", () => {
|
||||
@@ -40,57 +57,112 @@ describe("ChangePasswordModal", () => {
|
||||
memory: "test",
|
||||
platform: "test",
|
||||
runtime: "test",
|
||||
info: { isEnvCreds: false }
|
||||
info: { isEnvCreds: false },
|
||||
userInfo: { isIAMUser: false }
|
||||
}
|
||||
|
||||
it("should render without crashing", () => {
|
||||
shallow(<ChangePasswordModal serverInfo={serverInfo} />)
|
||||
})
|
||||
|
||||
it("should get the keys when its rendered", () => {
|
||||
const wrapper = shallow(<ChangePasswordModal serverInfo={serverInfo} />)
|
||||
setImmediate(() => {
|
||||
expect(wrapper.state("accessKey")).toBe("test1")
|
||||
expect(wrapper.state("secretKey")).toBe("test2")
|
||||
})
|
||||
it("should not allow changing password when isWorm is true", () => {
|
||||
const newServerInfo = { ...serverInfo, info: { isWorm: true } }
|
||||
const wrapper = shallow(<ChangePasswordModal serverInfo={newServerInfo} />)
|
||||
expect(
|
||||
wrapper
|
||||
.find("ModalBody")
|
||||
.childAt(0)
|
||||
.text()
|
||||
).toBe("Credentials of this user cannot be updated through MinIO Browser.")
|
||||
})
|
||||
|
||||
it("should show readonly keys when isEnvCreds is true", () => {
|
||||
const newServerInfo = { ...serverInfo, info: { isEnvCreds: true } }
|
||||
it("should not allow changing password when isEnvCreds is true and not IAM user", () => {
|
||||
const newServerInfo = {
|
||||
...serverInfo,
|
||||
info: { isEnvCreds: true },
|
||||
userInfo: { isIAMUser: false }
|
||||
}
|
||||
const wrapper = shallow(<ChangePasswordModal serverInfo={newServerInfo} />)
|
||||
expect(wrapper.state("accessKey")).toBe("xxxxxxxxx")
|
||||
expect(wrapper.state("secretKey")).toBe("xxxxxxxxx")
|
||||
expect(wrapper.find("#accessKey").prop("readonly")).toBeTruthy()
|
||||
expect(wrapper.find("#secretKey").prop("readonly")).toBeTruthy()
|
||||
expect(wrapper.find("#generate-keys").hasClass("hidden")).toBeTruthy()
|
||||
expect(wrapper.find("#update-keys").hasClass("hidden")).toBeTruthy()
|
||||
expect(
|
||||
wrapper
|
||||
.find("ModalBody")
|
||||
.childAt(0)
|
||||
.text()
|
||||
).toBe("Credentials of this user cannot be updated through MinIO Browser.")
|
||||
})
|
||||
|
||||
it("should generate accessKey and secretKey when Generate buttons is clicked", () => {
|
||||
const wrapper = shallow(<ChangePasswordModal serverInfo={serverInfo} />)
|
||||
wrapper.find("#generate-keys").simulate("click")
|
||||
setImmediate(() => {
|
||||
expect(wrapper.state("accessKey")).toBe("gen1")
|
||||
expect(wrapper.state("secretKey")).toBe("gen2")
|
||||
expect(wrapper.state("newAccessKey")).toBe("raccesskey")
|
||||
expect(wrapper.state("newSecretKey")).toBe("rsecretkey")
|
||||
})
|
||||
})
|
||||
|
||||
it("should not generate accessKey for IAM User", () => {
|
||||
const newServerInfo = {
|
||||
...serverInfo,
|
||||
userInfo: { isIAMUser: true }
|
||||
}
|
||||
const wrapper = shallow(<ChangePasswordModal serverInfo={newServerInfo} />)
|
||||
wrapper.find("#generate-keys").simulate("click")
|
||||
setImmediate(() => {
|
||||
expect(wrapper.state("newAccessKey")).toBe("minio")
|
||||
expect(wrapper.state("newSecretKey")).toBe("rsecretkey")
|
||||
})
|
||||
})
|
||||
|
||||
it("should not show new accessKey field for IAM User", () => {
|
||||
const newServerInfo = {
|
||||
...serverInfo,
|
||||
userInfo: { isIAMUser: true }
|
||||
}
|
||||
const wrapper = shallow(<ChangePasswordModal serverInfo={newServerInfo} />)
|
||||
expect(wrapper.find("#newAccesskey").exists()).toBeFalsy()
|
||||
})
|
||||
|
||||
it("should disble Update button for invalid accessKey or secretKey", () => {
|
||||
const showAlert = jest.fn()
|
||||
const wrapper = shallow(
|
||||
<ChangePasswordModal serverInfo={serverInfo} showAlert={showAlert} />
|
||||
)
|
||||
wrapper
|
||||
.find("#currentAccessKey")
|
||||
.simulate("change", { target: { value: "minio" } })
|
||||
wrapper
|
||||
.find("#currentSecretKey")
|
||||
.simulate("change", { target: { value: "minio123" } })
|
||||
wrapper.find("#newAccessKey").simulate("change", { target: { value: "t" } })
|
||||
wrapper
|
||||
.find("#newSecretKey")
|
||||
.simulate("change", { target: { value: "t1" } })
|
||||
expect(wrapper.find("#update-keys").prop("disabled")).toBeTruthy()
|
||||
})
|
||||
|
||||
it("should update accessKey and secretKey when Update button is clicked", () => {
|
||||
const showAlert = jest.fn()
|
||||
const wrapper = shallow(
|
||||
<ChangePasswordModal serverInfo={serverInfo} showAlert={showAlert} />
|
||||
)
|
||||
wrapper
|
||||
.find("#accessKey")
|
||||
.simulate("change", { target: { value: "test3" } })
|
||||
.find("#currentAccessKey")
|
||||
.simulate("change", { target: { value: "minio" } })
|
||||
wrapper
|
||||
.find("#secretKey")
|
||||
.simulate("change", { target: { value: "test4" } })
|
||||
.find("#currentSecretKey")
|
||||
.simulate("change", { target: { value: "minio123" } })
|
||||
wrapper
|
||||
.find("#newAccessKey")
|
||||
.simulate("change", { target: { value: "test" } })
|
||||
wrapper
|
||||
.find("#newSecretKey")
|
||||
.simulate("change", { target: { value: "test1234" } })
|
||||
expect(wrapper.find("#update-keys").prop("disabled")).toBeFalsy()
|
||||
wrapper.find("#update-keys").simulate("click")
|
||||
setImmediate(() => {
|
||||
expect(showAlert).toHaveBeenCalledWith({
|
||||
type: "success",
|
||||
message: "Changed credentials"
|
||||
message: "Credentials updated successfully."
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -34,7 +34,7 @@ export const fetchStorageInfo = () => {
|
||||
return web.StorageInfo().then(res => {
|
||||
const storageInfo = {
|
||||
total: res.storageInfo.Total,
|
||||
used: res.storageInfo.Used
|
||||
used: res.storageInfo.Used
|
||||
}
|
||||
dispatch(setStorageInfo(storageInfo))
|
||||
})
|
||||
@@ -54,7 +54,8 @@ export const fetchServerInfo = () => {
|
||||
memory: res.MinioMemory,
|
||||
platform: res.MinioPlatform,
|
||||
runtime: res.MinioRuntime,
|
||||
info: res.MinioGlobalInfo
|
||||
info: res.MinioGlobalInfo,
|
||||
userInfo: res.MinioUserInfo
|
||||
}
|
||||
dispatch(setServerInfo(serverInfo))
|
||||
})
|
||||
|
||||
@@ -28,3 +28,6 @@ export const NONE = "none"
|
||||
export const SHARE_OBJECT_EXPIRY_DAYS = 5
|
||||
export const SHARE_OBJECT_EXPIRY_HOURS = 0
|
||||
export const SHARE_OBJECT_EXPIRY_MINUTES = 0
|
||||
|
||||
export const ACCESS_KEY_MIN_LENGTH = 3
|
||||
export const SECRET_KEY_MIN_LENGTH = 8
|
||||
|
||||
@@ -208,7 +208,8 @@ export const shareObject = (object, days, hours, minutes) => {
|
||||
.PresignedGet({
|
||||
host: location.host,
|
||||
bucket: currentBucket,
|
||||
object: objectName
|
||||
object: objectName,
|
||||
expiry: expiry
|
||||
})
|
||||
.then(obj => {
|
||||
dispatch(showShareObject(object, obj.url))
|
||||
|
||||
@@ -14,11 +14,11 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { minioBrowserPrefix } from './constants.js'
|
||||
import { minioBrowserPrefix } from "./constants.js"
|
||||
|
||||
export const sortObjectsByName = (objects, order) => {
|
||||
let folders = objects.filter(object => object.name.endsWith('/'))
|
||||
let files = objects.filter(object => !object.name.endsWith('/'))
|
||||
let folders = objects.filter(object => object.name.endsWith("/"))
|
||||
let files = objects.filter(object => !object.name.endsWith("/"))
|
||||
folders = folders.sort((a, b) => {
|
||||
if (a.name.toLowerCase() < b.name.toLowerCase()) return -1
|
||||
if (a.name.toLowerCase() > b.name.toLowerCase()) return 1
|
||||
@@ -37,32 +37,34 @@ export const sortObjectsByName = (objects, order) => {
|
||||
}
|
||||
|
||||
export const sortObjectsBySize = (objects, order) => {
|
||||
let folders = objects.filter(object => object.name.endsWith('/'))
|
||||
let files = objects.filter(object => !object.name.endsWith('/'))
|
||||
let folders = objects.filter(object => object.name.endsWith("/"))
|
||||
let files = objects.filter(object => !object.name.endsWith("/"))
|
||||
files = files.sort((a, b) => a.size - b.size)
|
||||
if (order)
|
||||
files = files.reverse()
|
||||
if (order) files = files.reverse()
|
||||
return [...folders, ...files]
|
||||
}
|
||||
|
||||
export const sortObjectsByDate = (objects, order) => {
|
||||
let folders = objects.filter(object => object.name.endsWith('/'))
|
||||
let files = objects.filter(object => !object.name.endsWith('/'))
|
||||
files = files.sort((a, b) => new Date(a.lastModified).getTime() - new Date(b.lastModified).getTime())
|
||||
if (order)
|
||||
files = files.reverse()
|
||||
let folders = objects.filter(object => object.name.endsWith("/"))
|
||||
let files = objects.filter(object => !object.name.endsWith("/"))
|
||||
files = files.sort(
|
||||
(a, b) =>
|
||||
new Date(a.lastModified).getTime() - new Date(b.lastModified).getTime()
|
||||
)
|
||||
if (order) files = files.reverse()
|
||||
return [...folders, ...files]
|
||||
}
|
||||
|
||||
export const pathSlice = (path) => {
|
||||
path = path.replace(minioBrowserPrefix, '')
|
||||
let prefix = ''
|
||||
let bucket = ''
|
||||
if (!path) return {
|
||||
export const pathSlice = path => {
|
||||
path = path.replace(minioBrowserPrefix, "")
|
||||
let prefix = ""
|
||||
let bucket = ""
|
||||
if (!path)
|
||||
return {
|
||||
bucket,
|
||||
prefix
|
||||
}
|
||||
let objectIndex = path.indexOf('/', 1)
|
||||
}
|
||||
let objectIndex = path.indexOf("/", 1)
|
||||
if (objectIndex == -1) {
|
||||
bucket = path.slice(1)
|
||||
return {
|
||||
@@ -79,7 +81,29 @@ export const pathSlice = (path) => {
|
||||
}
|
||||
|
||||
export const pathJoin = (bucket, prefix) => {
|
||||
if (!prefix)
|
||||
prefix = ''
|
||||
return minioBrowserPrefix + '/' + bucket + '/' + prefix
|
||||
if (!prefix) prefix = ""
|
||||
return minioBrowserPrefix + "/" + bucket + "/" + prefix
|
||||
}
|
||||
|
||||
export const getRandomAccessKey = () => {
|
||||
const alphaNumericTable = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
||||
let arr = new Uint8Array(20)
|
||||
window.crypto.getRandomValues(arr)
|
||||
const random = Array.prototype.map.call(arr, v => {
|
||||
const i = v % alphaNumericTable.length
|
||||
return alphaNumericTable.charAt(i)
|
||||
})
|
||||
return random.join("")
|
||||
}
|
||||
|
||||
export const getRandomSecretKey = () => {
|
||||
let arr = new Uint8Array(40)
|
||||
window.crypto.getRandomValues(arr)
|
||||
const binStr = Array.prototype.map
|
||||
.call(arr, v => {
|
||||
return String.fromCharCode(v)
|
||||
})
|
||||
.join("")
|
||||
const base64Str = btoa(binStr)
|
||||
return base64Str.replace(/\//g, "+").substr(0, 40)
|
||||
}
|
||||
|
||||
@@ -72,6 +72,9 @@ class Web {
|
||||
Logout() {
|
||||
storage.removeItem('token')
|
||||
}
|
||||
GetToken() {
|
||||
return storage.getItem('token')
|
||||
}
|
||||
ServerInfo() {
|
||||
return this.makeCall('ServerInfo')
|
||||
}
|
||||
@@ -99,12 +102,6 @@ class Web {
|
||||
RemoveObject(args) {
|
||||
return this.makeCall('RemoveObject', args)
|
||||
}
|
||||
GetAuth() {
|
||||
return this.makeCall('GetAuth')
|
||||
}
|
||||
GenerateAuth() {
|
||||
return this.makeCall('GenerateAuth')
|
||||
}
|
||||
SetAuth(args) {
|
||||
return this.makeCall('SetAuth', args)
|
||||
.then(res => {
|
||||
|
||||
@@ -190,8 +190,8 @@
|
||||
----------------------------*/
|
||||
.toggle-password {
|
||||
position: absolute;
|
||||
bottom: 30px;
|
||||
right: 35px;
|
||||
bottom: 0 ;
|
||||
right: 0;
|
||||
width: 30px;
|
||||
height: 30px;
|
||||
border: 1px solid #eee;
|
||||
@@ -206,6 +206,10 @@
|
||||
background: #eee;
|
||||
}
|
||||
}
|
||||
|
||||
.has-toggle-password {
|
||||
position: relative;
|
||||
}
|
||||
//--------------------------
|
||||
|
||||
|
||||
|
||||
@@ -68,6 +68,7 @@
|
||||
"humanize": "0.0.9",
|
||||
"identity-obj-proxy": "^3.0.0",
|
||||
"json-loader": "^0.5.4",
|
||||
"jwt-decode": "^2.2.0",
|
||||
"local-storage-fallback": "^4.0.2",
|
||||
"material-design-iconic-font": "^2.2.0",
|
||||
"mime-db": "^1.25.0",
|
||||
|
||||
@@ -4907,6 +4907,11 @@ jsprim@^1.2.2:
|
||||
json-schema "0.2.3"
|
||||
verror "1.10.0"
|
||||
|
||||
jwt-decode@^2.2.0:
|
||||
version "2.2.0"
|
||||
resolved "https://registry.yarnpkg.com/jwt-decode/-/jwt-decode-2.2.0.tgz#7d86bd56679f58ce6a84704a657dd392bba81a79"
|
||||
integrity sha1-fYa9VmefWM5qhHBKZX3TkruoGnk=
|
||||
|
||||
keycode@^2.1.2:
|
||||
version "2.1.9"
|
||||
resolved "https://registry.yarnpkg.com/keycode/-/keycode-2.1.9.tgz#964a23c54e4889405b4861a5c9f0480d45141dfa"
|
||||
|
||||
@@ -99,6 +99,8 @@ var (
|
||||
"enable": false,
|
||||
"brokers": null,
|
||||
"topic": "",
|
||||
"queueDir": "",
|
||||
"queueLimit": 0,
|
||||
"tls": {
|
||||
"enable": false,
|
||||
"skipVerify": false,
|
||||
|
||||
@@ -27,7 +27,7 @@ import (
|
||||
"github.com/aliyun/aliyun-oss-go-sdk/oss"
|
||||
"google.golang.org/api/googleapi"
|
||||
|
||||
minio "github.com/minio/minio-go"
|
||||
minio "github.com/minio/minio-go/v6"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
@@ -879,16 +879,6 @@ var errorCodes = errorCodeMap{
|
||||
Description: "Object name already exists as a directory.",
|
||||
HTTPStatusCode: http.StatusConflict,
|
||||
},
|
||||
ErrReadQuorum: {
|
||||
Code: "XMinioReadQuorum",
|
||||
Description: "Multiple disk failures, unable to reconstruct data.",
|
||||
HTTPStatusCode: http.StatusServiceUnavailable,
|
||||
},
|
||||
ErrWriteQuorum: {
|
||||
Code: "XMinioWriteQuorum",
|
||||
Description: "Multiple disks failures, unable to write data.",
|
||||
HTTPStatusCode: http.StatusServiceUnavailable,
|
||||
},
|
||||
ErrInvalidObjectName: {
|
||||
Code: "XMinioInvalidObjectName",
|
||||
Description: "Object name contains unsupported characters.",
|
||||
@@ -1534,7 +1524,7 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
apiErr = ErrKMSAuthFailure
|
||||
case errOperationTimedOut, context.Canceled, context.DeadlineExceeded:
|
||||
apiErr = ErrOperationTimedOut
|
||||
case errNetworkConnReset:
|
||||
case errDiskNotFound:
|
||||
apiErr = ErrSlowDown
|
||||
}
|
||||
|
||||
@@ -1593,9 +1583,9 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
case InvalidPart:
|
||||
apiErr = ErrInvalidPart
|
||||
case InsufficientWriteQuorum:
|
||||
apiErr = ErrWriteQuorum
|
||||
apiErr = ErrSlowDown
|
||||
case InsufficientReadQuorum:
|
||||
apiErr = ErrReadQuorum
|
||||
apiErr = ErrSlowDown
|
||||
case UnsupportedDelimiter:
|
||||
apiErr = ErrNotImplemented
|
||||
case InvalidMarkerPrefixCombination:
|
||||
|
||||
@@ -39,8 +39,8 @@ var toAPIErrorTests = []struct {
|
||||
{err: ObjectNameInvalid{}, errCode: ErrInvalidObjectName},
|
||||
{err: InvalidUploadID{}, errCode: ErrNoSuchUpload},
|
||||
{err: InvalidPart{}, errCode: ErrInvalidPart},
|
||||
{err: InsufficientReadQuorum{}, errCode: ErrReadQuorum},
|
||||
{err: InsufficientWriteQuorum{}, errCode: ErrWriteQuorum},
|
||||
{err: InsufficientReadQuorum{}, errCode: ErrSlowDown},
|
||||
{err: InsufficientWriteQuorum{}, errCode: ErrSlowDown},
|
||||
{err: UnsupportedDelimiter{}, errCode: ErrNotImplemented},
|
||||
{err: InvalidMarkerPrefixCombination{}, errCode: ErrNotImplemented},
|
||||
{err: InvalidUploadIDKeyCombination{}, errCode: ErrNotImplemented},
|
||||
|
||||
@@ -30,7 +30,7 @@ import (
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio-go/v6/pkg/set"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/dns"
|
||||
|
||||
@@ -29,7 +29,7 @@ import (
|
||||
etcd "github.com/coreos/etcd/clientv3"
|
||||
dns2 "github.com/miekg/dns"
|
||||
"github.com/minio/cli"
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio-go/v6/pkg/set"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/cmd/logger/target/console"
|
||||
"github.com/minio/minio/cmd/logger/target/http"
|
||||
|
||||
@@ -107,7 +107,7 @@ func readConfigEtcd(ctx context.Context, client *etcd.Client, configFile string)
|
||||
// watchConfigEtcd - watches for changes on `configFile` on etcd and loads them.
|
||||
func watchConfigEtcd(objAPI ObjectLayer, configFile string, loadCfgFn func(ObjectLayer) error) {
|
||||
for {
|
||||
watchCh := globalEtcdClient.Watch(context.Background(), iamConfigPrefix)
|
||||
watchCh := globalEtcdClient.Watch(context.Background(), configFile)
|
||||
select {
|
||||
case <-GlobalServiceDoneCh:
|
||||
return
|
||||
|
||||
@@ -325,7 +325,7 @@ func (s *serverConfig) TestNotificationTargets() error {
|
||||
if !v.Enable {
|
||||
continue
|
||||
}
|
||||
t, err := target.NewKafkaTarget(k, v)
|
||||
t, err := target.NewKafkaTarget(k, v, GlobalServiceDoneCh)
|
||||
if err != nil {
|
||||
return fmt.Errorf("kafka(%s): %s", k, err.Error())
|
||||
}
|
||||
@@ -667,7 +667,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
|
||||
|
||||
for id, args := range config.Notify.Kafka {
|
||||
if args.Enable {
|
||||
newTarget, err := target.NewKafkaTarget(id, args)
|
||||
newTarget, err := target.NewKafkaTarget(id, args, GlobalServiceDoneCh)
|
||||
if err != nil {
|
||||
logger.LogIf(context.Background(), err)
|
||||
continue
|
||||
|
||||
@@ -200,7 +200,7 @@ func TestValidateConfig(t *testing.T) {
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
|
||||
|
||||
// Test 16 - Test Kafka
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "kafka": { "1": { "enable": true, "brokers": null, "topic": "" } }}}`, false},
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "kafka": { "1": { "enable": true, "brokers": null, "topic": "", "queueDir": "", "queueLimit": 0 } }}}`, false},
|
||||
|
||||
// Test 17 - Test Webhook
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "webhook": { "1": { "enable": true, "endpoint": "" } }}}`, false},
|
||||
|
||||
@@ -884,7 +884,7 @@ type serverConfigV32 struct {
|
||||
} `json:"policy"`
|
||||
}
|
||||
|
||||
// serverConfigV33 is just like version '32', removes clientID from NATS and MQTT, and adds queueDir, queueLimit with MQTT.
|
||||
// serverConfigV33 is just like version '32', removes clientID from NATS and MQTT, and adds queueDir, queueLimit in MQTT and kafka.
|
||||
type serverConfigV33 struct {
|
||||
quick.Config `json:"-"` // ignore interfaces
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@ import (
|
||||
"path"
|
||||
"strconv"
|
||||
|
||||
"github.com/minio/minio-go/pkg/encrypt"
|
||||
"github.com/minio/minio-go/v6/pkg/encrypt"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/ioutil"
|
||||
|
||||
@@ -23,7 +23,7 @@ import (
|
||||
"testing"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/minio/minio-go/pkg/encrypt"
|
||||
"github.com/minio/minio-go/v6/pkg/encrypt"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
"github.com/minio/sio"
|
||||
)
|
||||
|
||||
@@ -22,7 +22,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio-go/v6/pkg/set"
|
||||
"github.com/minio/minio/pkg/ellipses"
|
||||
)
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ import (
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/minio/cli"
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio-go/v6/pkg/set"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/cpu"
|
||||
"github.com/minio/minio/pkg/disk"
|
||||
|
||||
@@ -30,7 +30,7 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/pkg/s3utils"
|
||||
"github.com/minio/minio-go/v6/pkg/s3utils"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/lock"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
|
||||
@@ -24,7 +24,7 @@ import (
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
|
||||
minio "github.com/minio/minio-go"
|
||||
minio "github.com/minio/minio-go/v6"
|
||||
)
|
||||
|
||||
var (
|
||||
|
||||
@@ -221,6 +221,11 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
logger.FatalIf(err, "Unable to initialize gateway backend")
|
||||
}
|
||||
|
||||
// Populate existing buckets to the etcd backend
|
||||
if globalDNSConfig != nil {
|
||||
initFederatorBackend(newObject)
|
||||
}
|
||||
|
||||
if enableConfigOps {
|
||||
// Create a new config system.
|
||||
globalConfigSys = NewConfigSys()
|
||||
|
||||
@@ -36,7 +36,7 @@ import (
|
||||
"github.com/Azure/go-autorest/autorest/azure"
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/minio/cli"
|
||||
miniogopolicy "github.com/minio/minio-go/pkg/policy"
|
||||
miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
|
||||
"github.com/minio/minio/cmd"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
@@ -90,23 +90,23 @@ ENVIRONMENT VARIABLES:
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for Azure Blob Storage backend.
|
||||
$ export MINIO_ACCESS_KEY=azureaccountname
|
||||
$ export MINIO_SECRET_KEY=azureaccountkey
|
||||
$ {{.HelpName}}
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}azureaccountname
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}azureaccountkey
|
||||
{{.Prompt}} {{.HelpName}}
|
||||
|
||||
2. Start minio gateway server for Azure Blob Storage backend on custom endpoint.
|
||||
$ export MINIO_ACCESS_KEY=azureaccountname
|
||||
$ export MINIO_SECRET_KEY=azureaccountkey
|
||||
$ {{.HelpName}} https://azureaccountname.blob.custom.azure.endpoint
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}azureaccountname
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}azureaccountkey
|
||||
{{.Prompt}} {{.HelpName}} https://azureaccountname.blob.custom.azure.endpoint
|
||||
|
||||
3. Start minio gateway server for Azure Blob Storage backend with edge caching enabled.
|
||||
$ export MINIO_ACCESS_KEY=azureaccountname
|
||||
$ export MINIO_SECRET_KEY=azureaccountkey
|
||||
$ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
|
||||
$ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
|
||||
$ export MINIO_CACHE_EXPIRY=40
|
||||
$ export MINIO_CACHE_MAXUSE=80
|
||||
$ {{.HelpName}}
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}azureaccountname
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}azureaccountkey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png"
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_MAXUSE{{.AssignmentOperator}}80
|
||||
{{.Prompt}} {{.HelpName}}
|
||||
`
|
||||
|
||||
minio.RegisterGatewayCommand(cli.Command{
|
||||
|
||||
@@ -31,7 +31,7 @@ import (
|
||||
|
||||
b2 "github.com/minio/blazer/base"
|
||||
"github.com/minio/cli"
|
||||
miniogopolicy "github.com/minio/minio-go/pkg/policy"
|
||||
miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
h2 "github.com/minio/minio/pkg/hash"
|
||||
@@ -77,18 +77,18 @@ ENVIRONMENT VARIABLES:
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for B2 backend.
|
||||
$ export MINIO_ACCESS_KEY=accountID
|
||||
$ export MINIO_SECRET_KEY=applicationKey
|
||||
$ {{.HelpName}}
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accountID
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}applicationKey
|
||||
{{.Prompt}} {{.HelpName}}
|
||||
|
||||
2. Start minio gateway server for B2 backend with edge caching enabled.
|
||||
$ export MINIO_ACCESS_KEY=accountID
|
||||
$ export MINIO_SECRET_KEY=applicationKey
|
||||
$ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
|
||||
$ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
|
||||
$ export MINIO_CACHE_EXPIRY=40
|
||||
$ export MINIO_CACHE_MAXUSE=80
|
||||
$ {{.HelpName}}
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accountID
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}applicationKey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png"
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_MAXUSE{{.AssignmentOperator}}80
|
||||
{{.Prompt}} {{.HelpName}}
|
||||
`
|
||||
minio.RegisterGatewayCommand(cli.Command{
|
||||
Name: b2Backend,
|
||||
|
||||
@@ -19,6 +19,7 @@ package gcs
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -37,7 +38,7 @@ import (
|
||||
"cloud.google.com/go/storage"
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/minio/cli"
|
||||
miniogopolicy "github.com/minio/minio-go/pkg/policy"
|
||||
miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/policy"
|
||||
@@ -128,21 +129,21 @@ ENVIRONMENT VARIABLES:
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for GCS backend.
|
||||
$ export GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials.json
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} GOOGLE_APPLICATION_CREDENTIALS{{.AssignmentOperator}}/path/to/credentials.json
|
||||
(Instructions to generate credentials : https://developers.google.com/identity/protocols/application-default-credentials)
|
||||
$ export MINIO_ACCESS_KEY=accesskey
|
||||
$ export MINIO_SECRET_KEY=secretkey
|
||||
$ {{.HelpName}} mygcsprojectid
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
|
||||
{{.Prompt}} {{.HelpName}} mygcsprojectid
|
||||
|
||||
2. Start minio gateway server for GCS backend with edge caching enabled.
|
||||
$ export GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials.json
|
||||
$ export MINIO_ACCESS_KEY=accesskey
|
||||
$ export MINIO_SECRET_KEY=secretkey
|
||||
$ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
|
||||
$ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
|
||||
$ export MINIO_CACHE_EXPIRY=40
|
||||
$ export MINIO_CACHE_MAXUSE=80
|
||||
$ {{.HelpName}} mygcsprojectid
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} GOOGLE_APPLICATION_CREDENTIALS{{.AssignmentOperator}}/path/to/credentials.json
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png"
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_MAXUSE{{.AssignmentOperator}}80
|
||||
{{.Prompt}} {{.HelpName}} mygcsprojectid
|
||||
`
|
||||
|
||||
minio.RegisterGatewayCommand(cli.Command{
|
||||
@@ -830,12 +831,17 @@ func fromGCSAttrsToObjectInfo(attrs *storage.ObjectAttrs) minio.ObjectInfo {
|
||||
if attrs.ContentLanguage != "" {
|
||||
metadata["Content-Language"] = attrs.ContentLanguage
|
||||
}
|
||||
|
||||
etag := hex.EncodeToString(attrs.MD5)
|
||||
if etag == "" {
|
||||
etag = minio.ToS3ETag(fmt.Sprintf("%d", attrs.CRC32C))
|
||||
}
|
||||
return minio.ObjectInfo{
|
||||
Name: attrs.Name,
|
||||
Bucket: attrs.Bucket,
|
||||
ModTime: attrs.Updated,
|
||||
Size: attrs.Size,
|
||||
ETag: minio.ToS3ETag(fmt.Sprintf("%d", attrs.CRC32C)),
|
||||
ETag: etag,
|
||||
UserDefined: metadata,
|
||||
ContentType: attrs.ContentType,
|
||||
ContentEncoding: attrs.ContentEncoding,
|
||||
|
||||
@@ -28,7 +28,7 @@ import (
|
||||
"cloud.google.com/go/storage"
|
||||
"google.golang.org/api/googleapi"
|
||||
|
||||
miniogo "github.com/minio/minio-go"
|
||||
miniogo "github.com/minio/minio-go/v6"
|
||||
minio "github.com/minio/minio/cmd"
|
||||
)
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ package hdfs
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio-go/pkg/s3utils"
|
||||
"github.com/minio/minio-go/v6/pkg/s3utils"
|
||||
minio "github.com/minio/minio/cmd"
|
||||
)
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ import (
|
||||
|
||||
"github.com/minio/cli"
|
||||
"github.com/minio/hdfs/v3"
|
||||
"github.com/minio/minio-go/pkg/s3utils"
|
||||
"github.com/minio/minio-go/v6/pkg/s3utils"
|
||||
minio "github.com/minio/minio/cmd"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
@@ -76,18 +76,18 @@ ENVIRONMENT VARIABLES:
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for HDFS backend.
|
||||
$ export MINIO_ACCESS_KEY=accesskey
|
||||
$ export MINIO_SECRET_KEY=secretkey
|
||||
$ {{.HelpName}} hdfs://namenode:8200
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
|
||||
{{.Prompt}} {{.HelpName}} hdfs://namenode:8200
|
||||
|
||||
2. Start minio gateway server for HDFS with edge caching enabled.
|
||||
$ export MINIO_ACCESS_KEY=accesskey
|
||||
$ export MINIO_SECRET_KEY=secretkey
|
||||
$ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
|
||||
$ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
|
||||
$ export MINIO_CACHE_EXPIRY=40
|
||||
$ export MINIO_CACHE_MAXUSE=80
|
||||
$ {{.HelpName}} hdfs://namenode:8200
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png"
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_MAXUSE{{.AssignmentOperator}}80
|
||||
{{.Prompt}} {{.HelpName}} hdfs://namenode:8200
|
||||
`
|
||||
|
||||
minio.RegisterGatewayCommand(cli.Command{
|
||||
|
||||
@@ -60,18 +60,18 @@ ENVIRONMENT VARIABLES:
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for NAS backend.
|
||||
$ export MINIO_ACCESS_KEY=accesskey
|
||||
$ export MINIO_SECRET_KEY=secretkey
|
||||
$ {{.HelpName}} /shared/nasvol
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
|
||||
{{.Prompt}} {{.HelpName}} /shared/nasvol
|
||||
|
||||
2. Start minio gateway server for NAS with edge caching enabled.
|
||||
$ export MINIO_ACCESS_KEY=accesskey
|
||||
$ export MINIO_SECRET_KEY=secretkey
|
||||
$ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
|
||||
$ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
|
||||
$ export MINIO_CACHE_EXPIRY=40
|
||||
$ export MINIO_CACHE_MAXUSE=80
|
||||
$ {{.HelpName}} /shared/nasvol
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png"
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_MAXUSE{{.AssignmentOperator}}80
|
||||
{{.Prompt}} {{.HelpName}} /shared/nasvol
|
||||
`
|
||||
|
||||
minio.RegisterGatewayCommand(cli.Command{
|
||||
|
||||
@@ -29,8 +29,8 @@ import (
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
|
||||
"github.com/minio/cli"
|
||||
miniogopolicy "github.com/minio/minio-go/pkg/policy"
|
||||
"github.com/minio/minio-go/pkg/s3utils"
|
||||
miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
|
||||
"github.com/minio/minio-go/v6/pkg/s3utils"
|
||||
minio "github.com/minio/minio/cmd"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
@@ -79,23 +79,23 @@ ENVIRONMENT VARIABLES:
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for Aliyun OSS backend.
|
||||
$ export MINIO_ACCESS_KEY=accesskey
|
||||
$ export MINIO_SECRET_KEY=secretkey
|
||||
$ {{.HelpName}}
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
|
||||
{{.Prompt}} {{.HelpName}}
|
||||
|
||||
2. Start minio gateway server for Aliyun OSS backend on custom endpoint.
|
||||
$ export MINIO_ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
|
||||
$ export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
|
||||
$ {{.HelpName}} https://oss.example.com
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}Q3AM3UQ867SPQQA43P2F
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
|
||||
{{.Prompt}} {{.HelpName}} https://oss.example.com
|
||||
|
||||
3. Start minio gateway server for Aliyun OSS backend with edge caching enabled.
|
||||
$ export MINIO_ACCESS_KEY=accesskey
|
||||
$ export MINIO_SECRET_KEY=secretkey
|
||||
$ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
|
||||
$ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
|
||||
$ export MINIO_CACHE_EXPIRY=40
|
||||
$ export MINIO_CACHE_MAXUSE=80
|
||||
$ {{.HelpName}}
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png"
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_MAXUSE{{.AssignmentOperator}}80
|
||||
{{.Prompt}} {{.HelpName}}
|
||||
`
|
||||
|
||||
minio.RegisterGatewayCommand(cli.Command{
|
||||
|
||||
@@ -26,7 +26,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/pkg/encrypt"
|
||||
"github.com/minio/minio-go/v6/pkg/encrypt"
|
||||
minio "github.com/minio/minio/cmd"
|
||||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
|
||||
@@ -27,12 +27,12 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/minio/cli"
|
||||
miniogo "github.com/minio/minio-go"
|
||||
"github.com/minio/minio-go/pkg/credentials"
|
||||
miniogo "github.com/minio/minio-go/v6"
|
||||
"github.com/minio/minio-go/v6/pkg/credentials"
|
||||
minio "github.com/minio/minio/cmd"
|
||||
|
||||
"github.com/minio/minio-go/pkg/encrypt"
|
||||
"github.com/minio/minio-go/pkg/s3utils"
|
||||
"github.com/minio/minio-go/v6/pkg/encrypt"
|
||||
"github.com/minio/minio-go/v6/pkg/s3utils"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/policy"
|
||||
@@ -77,38 +77,38 @@ ENVIRONMENT VARIABLES:
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for AWS S3 backend.
|
||||
$ export MINIO_ACCESS_KEY=accesskey
|
||||
$ export MINIO_SECRET_KEY=secretkey
|
||||
$ {{.HelpName}}
|
||||
{{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}accesskey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}secretkey
|
||||
{{.Prompt}} {{.HelpName}}
|
||||
|
||||
2. Start minio gateway server for S3 backend on custom endpoint.
|
||||
$ export MINIO_ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
|
||||
$ export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
|
||||
$ {{.HelpName}} https://play.min.io:9000
|
||||
{{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}Q3AM3UQ867SPQQA43P2F
|
||||
{{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
|
||||
{{.Prompt}} {{.HelpName}} https://play.min.io:9000
|
||||
|
||||
3. Start minio gateway server for AWS S3 backend logging all requests to http endpoint.
|
||||
$ export MINIO_ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
|
||||
$ export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
|
||||
$ export MINIO_LOGGER_HTTP_ENDPOINT="http://localhost:8000/"
|
||||
$ {{.HelpName}} https://play.min.io:9000
|
||||
{{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}Q3AM3UQ867SPQQA43P2F
|
||||
{{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
|
||||
{{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}"http://localhost:8000/"
|
||||
{{.Prompt}} {{.HelpName}} https://play.min.io:9000
|
||||
|
||||
4. Start minio gateway server for AWS S3 backend with edge caching enabled.
|
||||
$ export MINIO_ACCESS_KEY=accesskey
|
||||
$ export MINIO_SECRET_KEY=secretkey
|
||||
$ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
|
||||
$ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
|
||||
$ export MINIO_CACHE_EXPIRY=40
|
||||
$ export MINIO_CACHE_MAXUSE=80
|
||||
$ {{.HelpName}}
|
||||
{{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}accesskey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}secretkey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}"/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
|
||||
{{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}"bucket1/*;*.png"
|
||||
{{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}40
|
||||
{{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}80
|
||||
{{.Prompt}} {{.HelpName}}
|
||||
|
||||
4. Start minio gateway server for AWS S3 backend using AWS environment variables.
|
||||
NOTE: The access and secret key in this case will authenticate with MinIO instead
|
||||
of AWS and AWS envs will be used to authenticate to AWS S3.
|
||||
$ export AWS_ACCESS_KEY_ID=aws_access_key
|
||||
$ export AWS_SECRET_ACCESS_KEY=aws_secret_key
|
||||
$ export MINIO_ACCESS_KEY=accesskey
|
||||
$ export MINIO_SECRET_KEY=secretkey
|
||||
$ {{.HelpName}}
|
||||
{{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}aws_access_key
|
||||
{{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}aws_secret_key
|
||||
{{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}accesskey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}secretkey
|
||||
{{.Prompt}} {{.HelpName}}
|
||||
`
|
||||
|
||||
minio.RegisterGatewayCommand(cli.Command{
|
||||
|
||||
@@ -20,7 +20,7 @@ import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
miniogo "github.com/minio/minio-go"
|
||||
miniogo "github.com/minio/minio-go/v6"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
|
||||
minio "github.com/minio/minio/cmd"
|
||||
|
||||
@@ -24,7 +24,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio-go/v6/pkg/set"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
@@ -379,6 +379,20 @@ type resourceHandler struct {
|
||||
|
||||
// setCorsHandler handler for CORS (Cross Origin Resource Sharing)
|
||||
func setCorsHandler(h http.Handler) http.Handler {
|
||||
commonS3Headers := []string{
|
||||
"Date",
|
||||
"ETag",
|
||||
"Server",
|
||||
"Connection",
|
||||
"Accept-Ranges",
|
||||
"Content-Range",
|
||||
"Content-Encoding",
|
||||
"Content-Length",
|
||||
"Content-Type",
|
||||
"X-Amz*",
|
||||
"x-amz*",
|
||||
"*",
|
||||
}
|
||||
|
||||
c := cors.New(cors.Options{
|
||||
AllowedOrigins: []string{"*"},
|
||||
@@ -391,8 +405,8 @@ func setCorsHandler(h http.Handler) http.Handler {
|
||||
http.MethodOptions,
|
||||
http.MethodPatch,
|
||||
},
|
||||
AllowedHeaders: []string{"*"},
|
||||
ExposedHeaders: []string{"*"},
|
||||
AllowedHeaders: commonS3Headers,
|
||||
ExposedHeaders: commonS3Headers,
|
||||
AllowCredentials: true,
|
||||
})
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ import (
|
||||
"time"
|
||||
|
||||
isatty "github.com/mattn/go-isatty"
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio-go/v6/pkg/set"
|
||||
|
||||
etcd "github.com/coreos/etcd/clientv3"
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
|
||||
@@ -28,7 +28,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio-go/v6/pkg/set"
|
||||
)
|
||||
|
||||
var serverPort uint32 = 60000
|
||||
|
||||
@@ -26,9 +26,8 @@ import (
|
||||
"time"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"golang.org/x/net/http2"
|
||||
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio-go/v6/pkg/set"
|
||||
"github.com/minio/minio/pkg/certs"
|
||||
)
|
||||
|
||||
@@ -56,9 +55,6 @@ const (
|
||||
|
||||
// DefaultMaxHeaderBytes - default maximum HTTP header size in bytes.
|
||||
DefaultMaxHeaderBytes = 1 * humanize.MiByte
|
||||
|
||||
// DefaultHTTP2MaxConcurrentStreams - default value for HTTP 2.0 maximum concurrent streams allowed.
|
||||
DefaultHTTP2MaxConcurrentStreams = 1024
|
||||
)
|
||||
|
||||
// Server - extended http.Server supports multiple addresses to serve and enhanced connection handling.
|
||||
@@ -133,11 +129,6 @@ func (srv *Server) Start() (err error) {
|
||||
|
||||
// Start servicing with listener.
|
||||
if tlsConfig != nil {
|
||||
if err = http2.ConfigureServer(&srv.Server, &http2.Server{
|
||||
MaxConcurrentStreams: DefaultHTTP2MaxConcurrentStreams,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return srv.Server.Serve(tls.NewListener(listener, tlsConfig))
|
||||
}
|
||||
return srv.Server.Serve(listener)
|
||||
@@ -211,7 +202,14 @@ func NewServer(addrs []string, handler http.Handler, getCert certs.GetCertificat
|
||||
CipherSuites: defaultCipherSuites,
|
||||
CurvePreferences: secureCurves,
|
||||
MinVersion: tls.VersionTLS12,
|
||||
NextProtos: []string{"h2", "http/1.1"},
|
||||
// Do not edit the next line, protos priority is kept
|
||||
// on purpose in this manner for HTTP 2.0, we would
|
||||
// still like HTTP 2.0 clients to negotiate connection
|
||||
// to server if needed but by default HTTP 1.1 is
|
||||
// expected. We need to change this in future
|
||||
// when we wish to go back to HTTP 2.0 as default
|
||||
// priority for HTTP protocol negotiation.
|
||||
NextProtos: []string{"http/1.1", "h2"},
|
||||
}
|
||||
tlsConfig.GetCertificate = getCert
|
||||
}
|
||||
|
||||
@@ -25,7 +25,7 @@ import (
|
||||
"time"
|
||||
|
||||
etcd "github.com/coreos/etcd/clientv3"
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio-go/v6/pkg/set"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
iampolicy "github.com/minio/minio/pkg/iam/policy"
|
||||
@@ -79,7 +79,8 @@ func (sys *IAMSys) Init(objAPI ObjectLayer) error {
|
||||
go func() {
|
||||
// Refresh IAMSys with etcd watch.
|
||||
for {
|
||||
watchCh := globalEtcdClient.Watch(context.Background(), iamConfigPrefix)
|
||||
watchCh := globalEtcdClient.Watch(context.Background(),
|
||||
iamConfigPrefix, etcd.WithPrefix())
|
||||
select {
|
||||
case <-GlobalServiceDoneCh:
|
||||
return
|
||||
@@ -490,6 +491,51 @@ func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetUserSecretKey - sets user secret key
|
||||
func (sys *IAMSys) SetUserSecretKey(accessKey string, secretKey string) error {
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil {
|
||||
return errServerNotInitialized
|
||||
}
|
||||
|
||||
sys.Lock()
|
||||
defer sys.Unlock()
|
||||
|
||||
cred, ok := sys.iamUsersMap[accessKey]
|
||||
if !ok {
|
||||
return errNoSuchUser
|
||||
}
|
||||
|
||||
uinfo := madmin.UserInfo{
|
||||
SecretKey: secretKey,
|
||||
Status: madmin.AccountStatus(cred.Status),
|
||||
}
|
||||
|
||||
configFile := pathJoin(iamConfigUsersPrefix, accessKey, iamIdentityFile)
|
||||
data, err := json.Marshal(uinfo)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if globalEtcdClient != nil {
|
||||
err = saveConfigEtcd(context.Background(), globalEtcdClient, configFile, data)
|
||||
} else {
|
||||
err = saveConfig(context.Background(), objectAPI, configFile, data)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sys.iamUsersMap[accessKey] = auth.Credentials{
|
||||
AccessKey: accessKey,
|
||||
SecretKey: secretKey,
|
||||
Status: string(uinfo.Status),
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetUser - get user credentials
|
||||
func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
|
||||
sys.RLock()
|
||||
|
||||
@@ -47,6 +47,7 @@ var (
|
||||
errChangeCredNotAllowed = errors.New("Changing access key and secret key not allowed")
|
||||
errAuthentication = errors.New("Authentication failed, check your access credentials")
|
||||
errNoAuthToken = errors.New("JWT token missing")
|
||||
errIncorrectCreds = errors.New("Current access key or secret key is incorrect")
|
||||
)
|
||||
|
||||
func authenticateJWTUsers(accessKey, secretKey string, expiry time.Duration) (string, error) {
|
||||
|
||||
@@ -17,12 +17,8 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
)
|
||||
|
||||
const lockRESTVersion = "v1"
|
||||
@@ -53,28 +49,22 @@ type lockResponse struct {
|
||||
func (l *localLocker) removeEntryIfExists(nlrip nameLockRequesterInfoPair) {
|
||||
// Check if entry is still in map (could have been removed altogether by 'concurrent' (R)Unlock of last entry)
|
||||
if lri, ok := l.lockMap[nlrip.name]; ok {
|
||||
if !l.removeEntry(nlrip.name, nlrip.lri.UID, &lri) {
|
||||
// Remove failed, in case it is a:
|
||||
if nlrip.lri.Writer {
|
||||
// Writer: this should never happen as the whole (mapped) entry should have been deleted
|
||||
reqInfo := (&logger.ReqInfo{}).AppendTags("name", nlrip.name)
|
||||
reqInfo.AppendTags("uid", nlrip.lri.UID)
|
||||
ctx := logger.SetReqInfo(context.Background(), reqInfo)
|
||||
logger.LogIf(ctx, errors.New("Lock maintenance failed to remove entry for write lock (should never happen)"))
|
||||
} // Reader: this can happen if multiple read locks were active and
|
||||
// the one we are looking for has been released concurrently (so it is fine).
|
||||
} // Removal went okay, all is fine.
|
||||
// Even if the entry exists, it may not be the same entry which was
|
||||
// considered as expired, so we simply an attempt to remove it if its
|
||||
// not possible there is nothing we need to do.
|
||||
l.removeEntry(nlrip.name, nlrip.lri.UID, &lri)
|
||||
}
|
||||
}
|
||||
|
||||
// removeEntry either, based on the uid of the lock message, removes a single entry from the
|
||||
// lockRequesterInfo array or the whole array from the map (in case of a write lock or last read lock)
|
||||
// removeEntry based on the uid of the lock message, removes a single entry from the
|
||||
// lockRequesterInfo array or the whole array from the map (in case of a write lock
|
||||
// or last read lock)
|
||||
func (l *localLocker) removeEntry(name, uid string, lri *[]lockRequesterInfo) bool {
|
||||
// Find correct entry to remove based on uid.
|
||||
for index, entry := range *lri {
|
||||
if entry.UID == uid {
|
||||
if len(*lri) == 1 {
|
||||
// Remove the (last) lock.
|
||||
// Remove the write lock.
|
||||
delete(l.lockMap, name)
|
||||
} else {
|
||||
// Remove the appropriate read lock.
|
||||
@@ -84,6 +74,7 @@ func (l *localLocker) removeEntry(name, uid string, lri *[]lockRequesterInfo) bo
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// None found return false, perhaps entry removed in previous run.
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -286,17 +286,17 @@ func (l *lockRESTServer) lockMaintenance(interval time.Duration) {
|
||||
Resource: nlrip.name,
|
||||
})
|
||||
|
||||
// Close the connection regardless of the call response.
|
||||
c.Close()
|
||||
|
||||
// For successful response, verify if lock is indeed active or stale.
|
||||
// For successful response, verify if lock was indeed active or stale.
|
||||
if expired {
|
||||
// The lock is no longer active at server that originated the lock
|
||||
// So remove the lock from the map.
|
||||
// The lock is no longer active at server that originated
|
||||
// the lock, attempt to remove the lock.
|
||||
l.ll.mutex.Lock()
|
||||
l.ll.removeEntryIfExists(nlrip) // Purge the stale entry if it exists.
|
||||
l.ll.mutex.Unlock()
|
||||
}
|
||||
|
||||
// Close the connection regardless of the call response.
|
||||
c.Close()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/minio/highwayhash"
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio-go/v6/pkg/set"
|
||||
"github.com/minio/minio/cmd/logger/message/log"
|
||||
)
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@ import (
|
||||
|
||||
"github.com/minio/dsync"
|
||||
"github.com/minio/lsync"
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio-go/v6/pkg/set"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
xnet "github.com/minio/minio/pkg/net"
|
||||
)
|
||||
|
||||
@@ -182,11 +182,11 @@ func (d *naughtyDisk) DeleteFileBulk(volume string, paths []string) ([]error, er
|
||||
return errs, nil
|
||||
}
|
||||
|
||||
func (d *naughtyDisk) WriteAll(volume string, path string, buf []byte) (err error) {
|
||||
func (d *naughtyDisk) WriteAll(volume string, path string, reader io.Reader) (err error) {
|
||||
if err := d.calcError(); err != nil {
|
||||
return err
|
||||
}
|
||||
return d.disk.WriteAll(volume, path, buf)
|
||||
return d.disk.WriteAll(volume, path, reader)
|
||||
}
|
||||
|
||||
func (d *naughtyDisk) ReadAll(volume string, path string) (buf []byte, err error) {
|
||||
|
||||
@@ -27,7 +27,7 @@ import (
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio-go/v6/pkg/set"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
)
|
||||
|
||||
|
||||
@@ -24,7 +24,7 @@ import (
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio-go/v6/pkg/set"
|
||||
)
|
||||
|
||||
func TestMustSplitHostPort(t *testing.T) {
|
||||
|
||||
@@ -31,6 +31,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/event"
|
||||
xnet "github.com/minio/minio/pkg/net"
|
||||
@@ -910,6 +911,11 @@ func (args eventArgs) ToEvent() event.Event {
|
||||
}
|
||||
|
||||
func sendEvent(args eventArgs) {
|
||||
|
||||
// remove sensitive encryption entries in metadata.
|
||||
crypto.RemoveSensitiveEntries(args.Object.UserDefined)
|
||||
crypto.RemoveInternalEntries(args.Object.UserDefined)
|
||||
|
||||
// globalNotificationSys is not initialized in gateway mode.
|
||||
if globalNotificationSys == nil {
|
||||
return
|
||||
|
||||
@@ -27,22 +27,6 @@ import (
|
||||
// underlying storage layer.
|
||||
func toObjectErr(err error, params ...string) error {
|
||||
switch err {
|
||||
case errDiskNotFound:
|
||||
switch len(params) {
|
||||
case 1:
|
||||
err = BucketNotFound{Bucket: params[0]}
|
||||
case 2:
|
||||
err = ObjectNotFound{
|
||||
Bucket: params[0],
|
||||
Object: params[1],
|
||||
}
|
||||
case 3:
|
||||
err = InvalidUploadID{
|
||||
Bucket: params[0],
|
||||
Object: params[1],
|
||||
UploadID: params[2],
|
||||
}
|
||||
}
|
||||
case errVolumeNotFound:
|
||||
if len(params) >= 1 {
|
||||
err = BucketNotFound{Bucket: params[0]}
|
||||
|
||||
@@ -21,7 +21,7 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/minio/minio-go/pkg/encrypt"
|
||||
"github.com/minio/minio-go/v6/pkg/encrypt"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
"github.com/minio/minio/pkg/policy"
|
||||
)
|
||||
|
||||
@@ -255,8 +255,8 @@ func testPutObjectPartDiskNotFound(obj ObjectLayer, instanceType string, disks [
|
||||
}
|
||||
|
||||
// As all disks at not available, bucket not found.
|
||||
expectedErr2 := BucketNotFound{Bucket: testCase.bucketName}
|
||||
if err.Error() != expectedErr2.Error() {
|
||||
expectedErr2 := errDiskNotFound
|
||||
if err != errDiskNotFound {
|
||||
t.Fatalf("Test %s: expected error %s, got %s instead.", instanceType, expectedErr2, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,7 +34,7 @@ import (
|
||||
"unicode/utf8"
|
||||
|
||||
snappy "github.com/golang/snappy"
|
||||
"github.com/minio/minio-go/pkg/s3utils"
|
||||
"github.com/minio/minio-go/v6/pkg/s3utils"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/dns"
|
||||
|
||||
@@ -33,8 +33,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
miniogo "github.com/minio/minio-go"
|
||||
"github.com/minio/minio-go/pkg/encrypt"
|
||||
miniogo "github.com/minio/minio-go/v6"
|
||||
"github.com/minio/minio-go/v6/pkg/encrypt"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/dns"
|
||||
|
||||
@@ -27,8 +27,8 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
miniogopolicy "github.com/minio/minio-go/pkg/policy"
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
|
||||
"github.com/minio/minio-go/v6/pkg/set"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/event"
|
||||
"github.com/minio/minio/pkg/handlers"
|
||||
|
||||
@@ -20,8 +20,8 @@ import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
miniogopolicy "github.com/minio/minio-go/pkg/policy"
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
|
||||
"github.com/minio/minio-go/v6/pkg/set"
|
||||
"github.com/minio/minio/pkg/policy"
|
||||
"github.com/minio/minio/pkg/policy/condition"
|
||||
)
|
||||
|
||||
@@ -35,20 +35,19 @@ import (
|
||||
"bytes"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/klauspost/readahead"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/disk"
|
||||
xioutil "github.com/minio/minio/pkg/ioutil"
|
||||
"github.com/minio/minio/pkg/mountinfo"
|
||||
"github.com/ncw/directio"
|
||||
)
|
||||
|
||||
const (
|
||||
diskMinFreeSpace = 900 * humanize.MiByte // Min 900MiB free space.
|
||||
diskMinTotalSpace = diskMinFreeSpace // Min 900MiB total space.
|
||||
maxAllowedIOError = 5
|
||||
posixWriteBlockSize = 4 * humanize.MiByte
|
||||
// DirectIO alignment needs to be 4K. Defined here as
|
||||
// directio.AlignSize is defined as 0 in MacOS causing divide by 0 error.
|
||||
directioAlignSize = 4096
|
||||
diskMinFreeSpace = 900 * humanize.MiByte // Min 900MiB free space.
|
||||
diskMinTotalSpace = diskMinFreeSpace // Min 900MiB total space.
|
||||
maxAllowedIOError = 5
|
||||
readBlockSize = humanize.KiByte * 32 // Default read block size 32KiB.
|
||||
)
|
||||
|
||||
// isValidVolname verifies a volname name in accordance with object
|
||||
@@ -190,10 +189,9 @@ func newPosix(path string) (*posix, error) {
|
||||
p := &posix{
|
||||
connected: true,
|
||||
diskPath: path,
|
||||
// 4MiB buffer pool for posix internal operations.
|
||||
pool: sync.Pool{
|
||||
New: func() interface{} {
|
||||
b := directio.AlignedBlock(posixWriteBlockSize)
|
||||
b := directio.AlignedBlock(readBlockSize)
|
||||
return &b
|
||||
},
|
||||
},
|
||||
@@ -1110,10 +1108,13 @@ func (s *posix) ReadFileStream(volume, path string, offset, length int64) (io.Re
|
||||
if _, err = file.Seek(offset, io.SeekStart); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return struct {
|
||||
|
||||
r := struct {
|
||||
io.Reader
|
||||
io.Closer
|
||||
}{Reader: io.LimitReader(file, length), Closer: file}, nil
|
||||
}{Reader: io.LimitReader(file, length), Closer: file}
|
||||
|
||||
return readahead.NewReadCloser(r), nil
|
||||
}
|
||||
|
||||
// CreateFile - creates the file.
|
||||
@@ -1182,8 +1183,6 @@ func (s *posix) CreateFile(volume, path string, fileSize int64, r io.Reader) (er
|
||||
return err
|
||||
}
|
||||
}
|
||||
defer w.Sync() // Sync before close.
|
||||
defer w.Close()
|
||||
|
||||
var e error
|
||||
if fileSize > 0 {
|
||||
@@ -1206,76 +1205,26 @@ func (s *posix) CreateFile(volume, path string, fileSize int64, r io.Reader) (er
|
||||
return err
|
||||
}
|
||||
|
||||
defer w.Close()
|
||||
|
||||
bufp := s.pool.Get().(*[]byte)
|
||||
defer s.pool.Put(bufp)
|
||||
|
||||
buf := *bufp
|
||||
|
||||
// Writes remaining bytes in the buffer.
|
||||
writeRemaining := func(w *os.File, buf []byte) (remainingWritten int, err error) {
|
||||
var n int
|
||||
remaining := len(buf)
|
||||
// The following logic writes the remainging data such that it writes whatever best is possible (aligned buffer)
|
||||
// in O_DIRECT mode and remaining (unaligned buffer) in non-O_DIRECT mode.
|
||||
remainingAligned := (remaining / directioAlignSize) * directioAlignSize
|
||||
remainingAlignedBuf := buf[:remainingAligned]
|
||||
remainingUnalignedBuf := buf[remainingAligned:]
|
||||
if len(remainingAlignedBuf) > 0 {
|
||||
n, err = w.Write(remainingAlignedBuf)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
remainingWritten += n
|
||||
}
|
||||
if len(remainingUnalignedBuf) > 0 {
|
||||
// Write on O_DIRECT fds fail if buffer is not 4K aligned, hence disable O_DIRECT.
|
||||
if err = disk.DisableDirectIO(w); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n, err = w.Write(remainingUnalignedBuf)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
remainingWritten += n
|
||||
}
|
||||
return remainingWritten, nil
|
||||
written, err := xioutil.CopyAligned(w, r, *bufp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var written int
|
||||
for {
|
||||
var n int
|
||||
n, err = io.ReadFull(r, buf)
|
||||
switch err {
|
||||
case nil:
|
||||
n, err = w.Write(buf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
written += n
|
||||
case io.ErrUnexpectedEOF:
|
||||
n, err = writeRemaining(w, buf[:n])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
written += n
|
||||
fallthrough
|
||||
case io.EOF:
|
||||
if fileSize != -1 {
|
||||
if written < int(fileSize) {
|
||||
return errLessData
|
||||
}
|
||||
if written > int(fileSize) {
|
||||
return errMoreData
|
||||
}
|
||||
}
|
||||
return nil
|
||||
default:
|
||||
return err
|
||||
}
|
||||
if written < fileSize {
|
||||
return errLessData
|
||||
} else if written > fileSize {
|
||||
return errMoreData
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *posix) WriteAll(volume, path string, buf []byte) (err error) {
|
||||
func (s *posix) WriteAll(volume, path string, reader io.Reader) (err error) {
|
||||
defer func() {
|
||||
if err == errFaultyDisk {
|
||||
atomic.AddInt32(&s.ioErrCount, 1)
|
||||
@@ -1293,11 +1242,13 @@ func (s *posix) WriteAll(volume, path string, buf []byte) (err error) {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err = w.Write(buf); err != nil {
|
||||
return err
|
||||
}
|
||||
defer w.Close()
|
||||
|
||||
return w.Close()
|
||||
bufp := s.pool.Get().(*[]byte)
|
||||
defer s.pool.Put(bufp)
|
||||
|
||||
_, err = io.CopyBuffer(w, reader, *bufp)
|
||||
return err
|
||||
}
|
||||
|
||||
// AppendFile - append a byte array at path, if file doesn't exist at
|
||||
|
||||
@@ -22,7 +22,7 @@ import (
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
minio "github.com/minio/minio-go"
|
||||
minio "github.com/minio/minio-go/v6"
|
||||
)
|
||||
|
||||
// Test Post Policy parsing and checking conditions
|
||||
|
||||
@@ -28,12 +28,22 @@ import (
|
||||
"time"
|
||||
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"golang.org/x/net/http2"
|
||||
)
|
||||
|
||||
// DefaultRESTTimeout - default RPC timeout is one minute.
|
||||
const DefaultRESTTimeout = 1 * time.Minute
|
||||
|
||||
// NetworkError - error type in case of errors related to http/transport
|
||||
// for ex. connection refused, connection reset, dns resolution failure etc.
|
||||
// All errors returned by storage-rest-server (ex errFileNotFound, errDiskNotFound) are not considered to be network errors.
|
||||
type NetworkError struct {
|
||||
Err error
|
||||
}
|
||||
|
||||
func (n *NetworkError) Error() string {
|
||||
return n.Err.Error()
|
||||
}
|
||||
|
||||
// Client - http based RPC client.
|
||||
type Client struct {
|
||||
httpClient *http.Client
|
||||
@@ -46,7 +56,7 @@ type Client struct {
|
||||
func (c *Client) Call(method string, values url.Values, body io.Reader, length int64) (reply io.ReadCloser, err error) {
|
||||
req, err := http.NewRequest(http.MethodPost, c.url.String()+"/"+method+"?"+values.Encode(), body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, &NetworkError{err}
|
||||
}
|
||||
|
||||
req.Header.Set("Authorization", "Bearer "+c.newAuthToken())
|
||||
@@ -56,7 +66,7 @@ func (c *Client) Call(method string, values url.Values, body io.Reader, length i
|
||||
}
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, &NetworkError{err}
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
@@ -100,20 +110,13 @@ func NewClient(url *url.URL, tlsConfig *tls.Config, timeout time.Duration, newAu
|
||||
tr := &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
DialContext: newCustomDialContext(timeout),
|
||||
MaxIdleConnsPerHost: 4096,
|
||||
MaxIdleConns: 4096,
|
||||
IdleConnTimeout: 120 * time.Second,
|
||||
MaxIdleConnsPerHost: 256,
|
||||
IdleConnTimeout: 60 * time.Second,
|
||||
TLSHandshakeTimeout: 30 * time.Second,
|
||||
ExpectContinueTimeout: 10 * time.Second,
|
||||
TLSClientConfig: tlsConfig,
|
||||
DisableCompression: true,
|
||||
}
|
||||
if tlsConfig != nil {
|
||||
// If TLS is enabled configure http2
|
||||
if err := http2.ConfigureTransport(tr); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return &Client{
|
||||
httpClient: &http.Client{Transport: tr},
|
||||
httpIdleConnsCloser: tr.CloseIdleConnections,
|
||||
|
||||
@@ -101,36 +101,36 @@ ENVIRONMENT VARIABLES:
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio server on "/home/shared" directory.
|
||||
$ {{.HelpName}} /home/shared
|
||||
{{.Prompt}} {{.HelpName}} /home/shared
|
||||
|
||||
2. Start minio server bound to a specific ADDRESS:PORT.
|
||||
$ {{.HelpName}} --address 192.168.1.101:9000 /home/shared
|
||||
{{.Prompt}} {{.HelpName}} --address 192.168.1.101:9000 /home/shared
|
||||
|
||||
3. Start minio server and enable virtual-host-style requests.
|
||||
$ export MINIO_DOMAIN=mydomain.com
|
||||
$ {{.HelpName}} --address mydomain.com:9000 /mnt/export
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_DOMAIN{{.AssignmentOperator}}mydomain.com
|
||||
{{.Prompt}} {{.HelpName}} --address mydomain.com:9000 /mnt/export
|
||||
|
||||
4. Start erasure coded minio server on a node with 64 drives.
|
||||
$ {{.HelpName}} /mnt/export{1...64}
|
||||
{{.Prompt}} {{.HelpName}} /mnt/export{1...64}
|
||||
|
||||
5. Start distributed minio server on an 32 node setup with 32 drives each. Run following command on all the 32 nodes.
|
||||
$ export MINIO_ACCESS_KEY=minio
|
||||
$ export MINIO_SECRET_KEY=miniostorage
|
||||
$ {{.HelpName}} http://node{1...32}.example.com/mnt/export/{1...32}
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}minio
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}miniostorage
|
||||
{{.Prompt}} {{.HelpName}} http://node{1...32}.example.com/mnt/export/{1...32}
|
||||
|
||||
6. Start minio server with edge caching enabled.
|
||||
$ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
|
||||
$ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
|
||||
$ export MINIO_CACHE_EXPIRY=40
|
||||
$ export MINIO_CACHE_MAXUSE=80
|
||||
$ {{.HelpName}} /home/shared
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png"
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_MAXUSE{{.AssignmentOperator}}80
|
||||
{{.Prompt}} {{.HelpName}} /home/shared
|
||||
|
||||
7. Start minio server with KMS enabled.
|
||||
$ export MINIO_SSE_VAULT_APPROLE_ID=9b56cc08-8258-45d5-24a3-679876769126
|
||||
$ export MINIO_SSE_VAULT_APPROLE_SECRET=4e30c52f-13e4-a6f5-0763-d50e8cb4321f
|
||||
$ export MINIO_SSE_VAULT_ENDPOINT=https://vault-endpoint-ip:8200
|
||||
$ export MINIO_SSE_VAULT_KEY_NAME=my-minio-key
|
||||
$ {{.HelpName}} /home/shared
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SSE_VAULT_APPROLE_ID{{.AssignmentOperator}}9b56cc08-8258-45d5-24a3-679876769126
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SSE_VAULT_APPROLE_SECRET{{.AssignmentOperator}}4e30c52f-13e4-a6f5-0763-d50e8cb4321f
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SSE_VAULT_ENDPOINT{{.AssignmentOperator}}https://vault-endpoint-ip:8200
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SSE_VAULT_KEY_NAME{{.AssignmentOperator}}my-minio-key
|
||||
{{.Prompt}} {{.HelpName}} /home/shared
|
||||
`,
|
||||
}
|
||||
|
||||
|
||||
@@ -146,6 +146,7 @@ func sumHMAC(key []byte, data []byte) []byte {
|
||||
// extractSignedHeaders extract signed headers from Authorization header
|
||||
func extractSignedHeaders(signedHeaders []string, r *http.Request) (http.Header, APIErrorCode) {
|
||||
reqHeaders := r.Header
|
||||
reqQueries := r.URL.Query()
|
||||
// find whether "host" is part of list of signed headers.
|
||||
// if not return ErrUnsignedHeaders. "host" is mandatory.
|
||||
if !contains(signedHeaders, "host") {
|
||||
@@ -156,6 +157,10 @@ func extractSignedHeaders(signedHeaders []string, r *http.Request) (http.Header,
|
||||
// `host` will not be found in the headers, can be found in r.Host.
|
||||
// but its alway necessary that the list of signed headers containing host in it.
|
||||
val, ok := reqHeaders[http.CanonicalHeaderKey(header)]
|
||||
if !ok {
|
||||
// try to set headers from Query String
|
||||
val, ok = reqQueries[header]
|
||||
}
|
||||
if ok {
|
||||
for _, enc := range val {
|
||||
extractedSignedHeaders.Add(header, enc)
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
@@ -149,6 +150,22 @@ func TestExtractSignedHeaders(t *testing.T) {
|
||||
t.Fatalf("Expected the APIErrorCode to be %d, but got %d", ErrNone, errCode)
|
||||
}
|
||||
|
||||
inputQuery := r.URL.Query()
|
||||
// case where some headers need to get from request query
|
||||
signedHeaders = append(signedHeaders, "x-amz-server-side-encryption")
|
||||
// expect to fail with `ErrUnsignedHeaders` because couldn't find some header
|
||||
_, errCode = extractSignedHeaders(signedHeaders, r)
|
||||
if errCode != ErrUnsignedHeaders {
|
||||
t.Fatalf("Expected the APIErrorCode to %d, but got %d", ErrUnsignedHeaders, errCode)
|
||||
}
|
||||
// set headers value through Get parameter
|
||||
inputQuery.Add("x-amz-server-side-encryption", crypto.SSEAlgorithmAES256)
|
||||
r.URL.RawQuery = inputQuery.Encode()
|
||||
_, errCode = extractSignedHeaders(signedHeaders, r)
|
||||
if errCode != ErrNone {
|
||||
t.Fatalf("Expected the APIErrorCode to be %d, but got %d", ErrNone, errCode)
|
||||
}
|
||||
|
||||
// "x-amz-content-sha256" header value from the extracted result.
|
||||
extractedContentSha256 := extractedSignedHeaders.Get("x-amz-content-sha256")
|
||||
// "host" header value from the extracted result.
|
||||
|
||||
@@ -35,7 +35,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/pkg/s3utils"
|
||||
"github.com/minio/minio-go/v6/pkg/s3utils"
|
||||
sha256 "github.com/minio/sha256-simd"
|
||||
)
|
||||
|
||||
@@ -251,13 +251,19 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
|
||||
|
||||
// Save other headers available in the request parameters.
|
||||
for k, v := range req.URL.Query() {
|
||||
key := strings.ToLower(k)
|
||||
|
||||
// Handle the metadata in presigned put query string
|
||||
if strings.Contains(strings.ToLower(k), "x-amz-meta-") {
|
||||
if strings.Contains(key, "x-amz-meta-") {
|
||||
query.Set(k, v[0])
|
||||
continue
|
||||
}
|
||||
|
||||
if strings.Contains(key, "x-amz-server-side-") {
|
||||
query.Set(k, v[0])
|
||||
}
|
||||
|
||||
if strings.HasPrefix(strings.ToLower(k), "x-amz") {
|
||||
if strings.HasPrefix(key, "x-amz") {
|
||||
continue
|
||||
}
|
||||
query[k] = v
|
||||
|
||||
@@ -54,7 +54,7 @@ type StorageAPI interface {
|
||||
DeleteFileBulk(volume string, paths []string) (errs []error, err error)
|
||||
|
||||
// Write all data, syncs the data to disk.
|
||||
WriteAll(volume string, path string, buf []byte) (err error)
|
||||
WriteAll(volume string, path string, reader io.Reader) (err error)
|
||||
|
||||
// Read all.
|
||||
ReadAll(volume string, path string) (buf []byte, err error)
|
||||
|
||||
@@ -21,7 +21,6 @@ import (
|
||||
"crypto/tls"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
@@ -44,32 +43,10 @@ func isNetworkError(err error) bool {
|
||||
if err.Error() == errConnectionStale.Error() {
|
||||
return true
|
||||
}
|
||||
if strings.Contains(err.Error(), "connection reset by peer") {
|
||||
if _, ok := err.(*rest.NetworkError); ok {
|
||||
return true
|
||||
}
|
||||
if uerr, isURLError := err.(*url.Error); isURLError {
|
||||
if uerr.Timeout() {
|
||||
return true
|
||||
}
|
||||
|
||||
err = uerr.Err
|
||||
}
|
||||
|
||||
_, isNetOpError := err.(*net.OpError)
|
||||
return isNetOpError
|
||||
}
|
||||
|
||||
// Attempt to approximate network error with a
|
||||
// typed network error, otherwise default to
|
||||
// errDiskNotFound
|
||||
func toNetworkError(err error) error {
|
||||
if err == nil {
|
||||
return err
|
||||
}
|
||||
if strings.Contains(err.Error(), "connection reset by peer") {
|
||||
return errNetworkConnReset
|
||||
}
|
||||
return errDiskNotFound
|
||||
return false
|
||||
}
|
||||
|
||||
// Converts rpc.ServerError to underlying error. This function is
|
||||
@@ -81,7 +58,7 @@ func toStorageErr(err error) error {
|
||||
}
|
||||
|
||||
if isNetworkError(err) {
|
||||
return toNetworkError(err)
|
||||
return errDiskNotFound
|
||||
}
|
||||
|
||||
switch err.Error() {
|
||||
@@ -256,11 +233,10 @@ func (client *storageRESTClient) CreateFile(volume, path string, length int64, r
|
||||
}
|
||||
|
||||
// WriteAll - write all data to a file.
|
||||
func (client *storageRESTClient) WriteAll(volume, path string, buffer []byte) error {
|
||||
func (client *storageRESTClient) WriteAll(volume, path string, reader io.Reader) error {
|
||||
values := make(url.Values)
|
||||
values.Set(storageRESTVolume, volume)
|
||||
values.Set(storageRESTFilePath, path)
|
||||
reader := bytes.NewBuffer(buffer)
|
||||
respBody, err := client.call(storageRESTMethodWriteAll, values, reader, -1)
|
||||
defer http.DrainBody(respBody)
|
||||
return err
|
||||
|
||||
@@ -245,14 +245,7 @@ func (s *storageRESTServer) WriteAllHandler(w http.ResponseWriter, r *http.Reque
|
||||
return
|
||||
}
|
||||
|
||||
buf := make([]byte, r.ContentLength)
|
||||
_, err := io.ReadFull(r.Body, buf)
|
||||
if err != nil {
|
||||
s.writeErrorResponse(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
err = s.storage.WriteAll(volume, filePath, buf)
|
||||
err := s.storage.WriteAll(volume, filePath, io.LimitReader(r.Body, r.ContentLength))
|
||||
if err != nil {
|
||||
s.writeErrorResponse(w, err)
|
||||
}
|
||||
@@ -415,7 +408,6 @@ func (s *storageRESTServer) WalkHandler(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
defer w.(http.Flusher).Flush()
|
||||
|
||||
w.Header().Set("Connection", "close") // Pro-actively ask client to close this connection.
|
||||
encoder := gob.NewEncoder(w)
|
||||
for fi := range fch {
|
||||
encoder.Encode(&fi)
|
||||
|
||||
@@ -54,8 +54,8 @@ import (
|
||||
|
||||
"github.com/fatih/color"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/minio-go/pkg/s3signer"
|
||||
"github.com/minio/minio-go/pkg/s3utils"
|
||||
"github.com/minio/minio-go/v6/pkg/s3signer"
|
||||
"github.com/minio/minio-go/v6/pkg/s3utils"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/bpool"
|
||||
@@ -174,7 +174,7 @@ func prepareFS() (ObjectLayer, string, error) {
|
||||
return obj, fsDirs[0], nil
|
||||
}
|
||||
|
||||
func prepareXL32() (ObjectLayer, []string, error) {
|
||||
func prepareXLSets32() (ObjectLayer, []string, error) {
|
||||
fsDirs1, err := getRandomDisks(16)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
@@ -1704,7 +1704,7 @@ func prepareTestBackend(instanceType string) (ObjectLayer, []string, error) {
|
||||
switch instanceType {
|
||||
// Total number of disks for XL sets backend is set to 32.
|
||||
case XLSetsTestStr:
|
||||
return prepareXL32()
|
||||
return prepareXLSets32()
|
||||
// Total number of disks for XL backend is set to 16.
|
||||
case XLTestStr:
|
||||
return prepareXL16()
|
||||
@@ -1986,7 +1986,7 @@ func ExecObjectLayerTest(t TestErrHandler, objTest objTestType) {
|
||||
// Executing the object layer tests for single node setup.
|
||||
objTest(objLayer, FSTestStr, t)
|
||||
|
||||
objLayer, fsDirs, err := prepareXL16()
|
||||
objLayer, fsDirs, err := prepareXLSets32()
|
||||
if err != nil {
|
||||
t.Fatalf("Initialization of object layer failed for XL setup: %s", err)
|
||||
}
|
||||
|
||||
@@ -85,6 +85,3 @@ var errNoSuchPolicy = errors.New("Specified canned policy does not exist")
|
||||
|
||||
// error returned when access is denied.
|
||||
var errAccessDenied = errors.New("Do not have enough permissions to access this resource")
|
||||
|
||||
// errNetworkConnReset - connection reset by peer
|
||||
var errNetworkConnReset = errors.New("connection reset by peer")
|
||||
|
||||
@@ -65,7 +65,7 @@ EXIT STATUS:
|
||||
|
||||
EXAMPLES:
|
||||
1. Check and update minio:
|
||||
$ {{.HelpName}}
|
||||
{{.Prompt}} {{.HelpName}}
|
||||
`,
|
||||
}
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ FLAGS:
|
||||
{{end}}{{end}}
|
||||
EXAMPLES:
|
||||
1. Prints server version:
|
||||
$ {{.HelpName}}
|
||||
{{.Prompt}} {{.HelpName}}
|
||||
`,
|
||||
}
|
||||
|
||||
|
||||
@@ -36,9 +36,9 @@ import (
|
||||
snappy "github.com/golang/snappy"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/gorilla/rpc/v2/json2"
|
||||
miniogopolicy "github.com/minio/minio-go/pkg/policy"
|
||||
"github.com/minio/minio-go/pkg/s3utils"
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
|
||||
"github.com/minio/minio-go/v6/pkg/s3utils"
|
||||
"github.com/minio/minio-go/v6/pkg/set"
|
||||
"github.com/minio/minio/browser"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
@@ -69,6 +69,7 @@ type ServerInfoRep struct {
|
||||
MinioPlatform string
|
||||
MinioRuntime string
|
||||
MinioGlobalInfo map[string]interface{}
|
||||
MinioUserInfo map[string]interface{}
|
||||
UIVersion string `json:"uiVersion"`
|
||||
}
|
||||
|
||||
@@ -97,17 +98,17 @@ func (web *webAPIHandlers) ServerInfo(r *http.Request, args *WebGenericArgs, rep
|
||||
|
||||
reply.MinioVersion = Version
|
||||
reply.MinioGlobalInfo = getGlobalInfo()
|
||||
// If ENV creds are not set and incoming user is not owner
|
||||
// disable changing credentials.
|
||||
v, ok := reply.MinioGlobalInfo["isEnvCreds"].(bool)
|
||||
if ok && !v {
|
||||
reply.MinioGlobalInfo["isEnvCreds"] = !owner
|
||||
}
|
||||
// if etcd is set disallow changing credentials through UI
|
||||
|
||||
// if etcd is set, disallow changing credentials through UI for owner
|
||||
if globalEtcdClient != nil {
|
||||
reply.MinioGlobalInfo["isEnvCreds"] = true
|
||||
}
|
||||
|
||||
// Check if the user is IAM user
|
||||
reply.MinioUserInfo = map[string]interface{}{
|
||||
"isIAMUser": !owner,
|
||||
}
|
||||
|
||||
reply.MinioMemory = mem
|
||||
reply.MinioPlatform = platform
|
||||
reply.MinioRuntime = goruntime
|
||||
@@ -768,8 +769,10 @@ func (web webAPIHandlers) GenerateAuth(r *http.Request, args *WebGenericArgs, re
|
||||
|
||||
// SetAuthArgs - argument for SetAuth
|
||||
type SetAuthArgs struct {
|
||||
AccessKey string `json:"accessKey"`
|
||||
SecretKey string `json:"secretKey"`
|
||||
CurrentAccessKey string `json:"currentAccessKey"`
|
||||
CurrentSecretKey string `json:"currentSecretKey"`
|
||||
NewAccessKey string `json:"newAccessKey"`
|
||||
NewSecretKey string `json:"newSecretKey"`
|
||||
}
|
||||
|
||||
// SetAuthReply - reply for SetAuth
|
||||
@@ -781,68 +784,85 @@ type SetAuthReply struct {
|
||||
|
||||
// SetAuth - Set accessKey and secretKey credentials.
|
||||
func (web *webAPIHandlers) SetAuth(r *http.Request, args *SetAuthArgs, reply *SetAuthReply) error {
|
||||
_, owner, authErr := webRequestAuthenticate(r)
|
||||
claims, owner, authErr := webRequestAuthenticate(r)
|
||||
if authErr != nil {
|
||||
return toJSONError(authErr)
|
||||
}
|
||||
|
||||
// If creds are set through ENV disallow changing credentials.
|
||||
if globalIsEnvCreds || globalWORMEnabled || !owner || globalEtcdClient != nil {
|
||||
// When WORM is enabled, disallow changing credenatials for owner and user
|
||||
if globalWORMEnabled {
|
||||
return toJSONError(errChangeCredNotAllowed)
|
||||
}
|
||||
|
||||
creds, err := auth.CreateCredentials(args.AccessKey, args.SecretKey)
|
||||
if err != nil {
|
||||
return toJSONError(err)
|
||||
if owner {
|
||||
if globalIsEnvCreds || globalEtcdClient != nil {
|
||||
return toJSONError(errChangeCredNotAllowed)
|
||||
}
|
||||
|
||||
// get Current creds and verify
|
||||
prevCred := globalServerConfig.GetCredential()
|
||||
if prevCred.AccessKey != args.CurrentAccessKey || prevCred.SecretKey != args.CurrentSecretKey {
|
||||
return errIncorrectCreds
|
||||
}
|
||||
|
||||
creds, err := auth.CreateCredentials(args.NewAccessKey, args.NewSecretKey)
|
||||
if err != nil {
|
||||
return toJSONError(err)
|
||||
}
|
||||
|
||||
// Acquire lock before updating global configuration.
|
||||
globalServerConfigMu.Lock()
|
||||
defer globalServerConfigMu.Unlock()
|
||||
|
||||
// Update credentials in memory
|
||||
prevCred = globalServerConfig.SetCredential(creds)
|
||||
|
||||
// Persist updated credentials.
|
||||
if err = saveServerConfig(context.Background(), newObjectLayerFn(), globalServerConfig); err != nil {
|
||||
// Save the current creds when failed to update.
|
||||
globalServerConfig.SetCredential(prevCred)
|
||||
logger.LogIf(context.Background(), err)
|
||||
return toJSONError(err)
|
||||
}
|
||||
|
||||
reply.Token, err = authenticateWeb(args.NewAccessKey, args.NewSecretKey)
|
||||
if err != nil {
|
||||
return toJSONError(err)
|
||||
}
|
||||
} else {
|
||||
// for IAM users, access key cannot be updated
|
||||
// claims.Subject is used instead of accesskey from args
|
||||
prevCred, ok := globalIAMSys.GetUser(claims.Subject)
|
||||
if !ok {
|
||||
return errInvalidAccessKeyID
|
||||
}
|
||||
|
||||
// Throw error when wrong secret key is provided
|
||||
if prevCred.SecretKey != args.CurrentSecretKey {
|
||||
return errIncorrectCreds
|
||||
}
|
||||
|
||||
creds, err := auth.CreateCredentials(claims.Subject, args.NewSecretKey)
|
||||
if err != nil {
|
||||
return toJSONError(err)
|
||||
}
|
||||
|
||||
err = globalIAMSys.SetUserSecretKey(creds.AccessKey, creds.SecretKey)
|
||||
if err != nil {
|
||||
return toJSONError(err)
|
||||
}
|
||||
|
||||
reply.Token, err = authenticateWeb(creds.AccessKey, creds.SecretKey)
|
||||
if err != nil {
|
||||
return toJSONError(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Acquire lock before updating global configuration.
|
||||
globalServerConfigMu.Lock()
|
||||
defer globalServerConfigMu.Unlock()
|
||||
|
||||
// Update credentials in memory
|
||||
prevCred := globalServerConfig.SetCredential(creds)
|
||||
|
||||
// Persist updated credentials.
|
||||
if err = saveServerConfig(context.Background(), newObjectLayerFn(), globalServerConfig); err != nil {
|
||||
// Save the current creds when failed to update.
|
||||
globalServerConfig.SetCredential(prevCred)
|
||||
logger.LogIf(context.Background(), err)
|
||||
return toJSONError(err)
|
||||
}
|
||||
|
||||
reply.Token, err = authenticateWeb(creds.AccessKey, creds.SecretKey)
|
||||
if err != nil {
|
||||
return toJSONError(err)
|
||||
}
|
||||
reply.UIVersion = browser.UIVersion
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetAuthReply - Reply current credentials.
|
||||
type GetAuthReply struct {
|
||||
AccessKey string `json:"accessKey"`
|
||||
SecretKey string `json:"secretKey"`
|
||||
UIVersion string `json:"uiVersion"`
|
||||
}
|
||||
|
||||
// GetAuth - return accessKey and secretKey credentials.
|
||||
func (web *webAPIHandlers) GetAuth(r *http.Request, args *WebGenericArgs, reply *GetAuthReply) error {
|
||||
_, owner, authErr := webRequestAuthenticate(r)
|
||||
if authErr != nil {
|
||||
return toJSONError(authErr)
|
||||
}
|
||||
if !owner {
|
||||
return toJSONError(errAccessDenied)
|
||||
}
|
||||
creds := globalServerConfig.GetCredential()
|
||||
reply.AccessKey = creds.AccessKey
|
||||
reply.SecretKey = creds.SecretKey
|
||||
reply.UIVersion = browser.UIVersion
|
||||
return nil
|
||||
}
|
||||
|
||||
// URLTokenReply contains the reply for CreateURLToken.
|
||||
type URLTokenReply struct {
|
||||
Token string `json:"token"`
|
||||
|
||||
@@ -35,7 +35,7 @@ import (
|
||||
|
||||
jwtgo "github.com/dgrijalva/jwt-go"
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
miniogopolicy "github.com/minio/minio-go/pkg/policy"
|
||||
miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
"github.com/minio/minio/pkg/policy"
|
||||
"github.com/minio/minio/pkg/policy/condition"
|
||||
@@ -701,18 +701,21 @@ func testSetAuthWebHandler(obj ObjectLayer, instanceType string, t TestErrHandle
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
username string
|
||||
password string
|
||||
success bool
|
||||
currentAccessKey string
|
||||
currentSecretKey string
|
||||
newAccessKey string
|
||||
newSecretKey string
|
||||
success bool
|
||||
}{
|
||||
{"", "", false},
|
||||
{"1", "1", false},
|
||||
{"azerty", "foooooooooooooo", true},
|
||||
{"", "", "", "", false},
|
||||
{"1", "1", "1", "1", false},
|
||||
{credentials.AccessKey, credentials.SecretKey, "azerty", "bar", false},
|
||||
{credentials.AccessKey, credentials.SecretKey, "azerty", "foooooooooooooo", true},
|
||||
}
|
||||
|
||||
// Iterating over the test cases, calling the function under test and asserting the response.
|
||||
for i, testCase := range testCases {
|
||||
setAuthRequest := SetAuthArgs{AccessKey: testCase.username, SecretKey: testCase.password}
|
||||
setAuthRequest := SetAuthArgs{CurrentAccessKey: testCase.currentAccessKey, CurrentSecretKey: testCase.currentSecretKey, NewAccessKey: testCase.newAccessKey, NewSecretKey: testCase.newSecretKey}
|
||||
setAuthReply := &SetAuthReply{}
|
||||
req, err := newTestWebRPCRequest("Web.SetAuth", authorization, setAuthRequest)
|
||||
if err != nil {
|
||||
@@ -735,42 +738,6 @@ func testSetAuthWebHandler(obj ObjectLayer, instanceType string, t TestErrHandle
|
||||
}
|
||||
}
|
||||
|
||||
// Wrapper for calling Get Auth Handler
|
||||
func TestWebHandlerGetAuth(t *testing.T) {
|
||||
ExecObjectLayerTest(t, testGetAuthWebHandler)
|
||||
}
|
||||
|
||||
// testGetAuthWebHandler - Test GetAuth web handler
|
||||
func testGetAuthWebHandler(obj ObjectLayer, instanceType string, t TestErrHandler) {
|
||||
// Register the API end points with XL/FS object layer.
|
||||
apiRouter := initTestWebRPCEndPoint(obj)
|
||||
credentials := globalServerConfig.GetCredential()
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
authorization, err := getWebRPCToken(apiRouter, credentials.AccessKey, credentials.SecretKey)
|
||||
if err != nil {
|
||||
t.Fatal("Cannot authenticate")
|
||||
}
|
||||
|
||||
getAuthRequest := WebGenericArgs{}
|
||||
getAuthReply := &GetAuthReply{}
|
||||
req, err := newTestWebRPCRequest("Web.GetAuth", authorization, getAuthRequest)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create HTTP request: <ERROR> %v", err)
|
||||
}
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("Expected the response status to be 200, but instead found `%d`", rec.Code)
|
||||
}
|
||||
err = getTestWebRPCResponse(rec, &getAuthReply)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed, %v", err)
|
||||
}
|
||||
if getAuthReply.AccessKey != credentials.AccessKey || getAuthReply.SecretKey != credentials.SecretKey {
|
||||
t.Fatalf("Failed to get correct auth keys")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebCreateURLToken(t *testing.T) {
|
||||
ExecObjectLayerTest(t, testCreateURLToken)
|
||||
}
|
||||
@@ -1518,7 +1485,7 @@ func TestWebCheckAuthorization(t *testing.T) {
|
||||
webRPCs := []string{
|
||||
"ServerInfo", "StorageInfo", "MakeBucket",
|
||||
"ListBuckets", "ListObjects", "RemoveObject",
|
||||
"GenerateAuth", "SetAuth", "GetAuth",
|
||||
"GenerateAuth", "SetAuth",
|
||||
"GetBucketPolicy", "SetBucketPolicy", "ListAllBucketPolicies",
|
||||
"PresignedGet",
|
||||
}
|
||||
|
||||
@@ -858,10 +858,21 @@ func leastEntry(entriesCh []FileInfoCh, readQuorum int) (FileInfo, bool) {
|
||||
|
||||
// mergeEntriesCh - merges FileInfo channel to entries upto maxKeys.
|
||||
func mergeEntriesCh(entriesCh []FileInfoCh, maxKeys int, readQuorum int) (entries FilesInfo) {
|
||||
for i := 0; i < maxKeys; {
|
||||
var fi FileInfo
|
||||
fi, entries.IsTruncated = leastEntry(entriesCh, readQuorum)
|
||||
if !entries.IsTruncated {
|
||||
var i = 0
|
||||
for {
|
||||
fi, valid := leastEntry(entriesCh, readQuorum)
|
||||
if !valid {
|
||||
break
|
||||
}
|
||||
if i == maxKeys {
|
||||
entries.IsTruncated = true
|
||||
// Re-insert the last entry so it can be
|
||||
// listed in the next listing iteration.
|
||||
for j := range entriesCh {
|
||||
if !entriesCh[j].Valid {
|
||||
entriesCh[j].Push(fi)
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
entries.Files = append(entries.Files, fi)
|
||||
@@ -944,7 +955,6 @@ func (s *xlSets) ListObjects(ctx context.Context, bucket, prefix, marker, delimi
|
||||
return loi, nil
|
||||
}
|
||||
|
||||
loi.Objects = make([]ObjectInfo, len(entries.Files))
|
||||
loi.IsTruncated = entries.IsTruncated
|
||||
if loi.IsTruncated {
|
||||
loi.NextMarker = entries.Files[len(entries.Files)-1].Name
|
||||
|
||||
@@ -21,7 +21,7 @@ import (
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"github.com/minio/minio-go/pkg/s3utils"
|
||||
"github.com/minio/minio-go/v6/pkg/s3utils"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/policy"
|
||||
)
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
@@ -437,7 +438,7 @@ func writeXLMetadata(ctx context.Context, disk StorageAPI, bucket, prefix string
|
||||
}
|
||||
|
||||
// Persist marshaled data.
|
||||
err = disk.WriteAll(bucket, jsonFile, metadataBytes)
|
||||
err = disk.WriteAll(bucket, jsonFile, bytes.NewReader(metadataBytes))
|
||||
logger.LogIf(ctx, err)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
# **Disaggregated HDP Spark and Hive with MinIO**
|
||||
|
||||
## **1. Cloud-native Architecture**
|
||||
|
||||

|
||||
|
||||
Kubernetes manages stateless Spark and Hive containers elastically on the compute nodes. Spark has native scheduler integration with Kubernetes. Hive, for legacy reasons, uses YARN scheduler on top of Kubernetes.
|
||||
|
||||
All access to MinIO object storage is via S3/SQL SELECT API. In addition to the compute nodes, MinIO containers are also managed by Kubernetes as stateful containers with local storage (JBOD/JBOF) mapped as persistent local volumes. This architecture enables multi-tenant MinIO, allowing isolation of data between customers.
|
||||
|
||||
MinIO also supports multi-cluster, multi-site federation similar to AWS regions and tiers. Using MinIO Information Lifecycle Management (ILM), you can configure data to be tiered between NVMe based hot storage, and HDD based warm storage. All data is encrypted with per-object key. Access Control and Identity Management between the tenants are managed by MinIO using OpenID Connect or Kerberos/LDAP/AD.
|
||||
|
||||
## **2. Prerequisites**
|
||||
|
||||
* Install Hortonworks Distribution using this [guide.](https://docs.hortonworks.com/HDPDocuments/Ambari-2.7.1.0/bk_ambari-installation/content/ch_Installing_Ambari.html)
|
||||
* [Setup Ambari](https://docs.hortonworks.com/HDPDocuments/Ambari-2.7.1.0/bk_ambari-installation/content/set_up_the_ambari_server.html) which automatically sets up YARN
|
||||
* [Installing Spark](https://docs.hortonworks.com/HDPDocuments/HDP3/HDP-3.0.1/installing-spark/content/installing_spark.html)
|
||||
* Install MinIO Distributed Server using one of the guides below.
|
||||
* [Deployment based on Kubernetes](https://docs.min.io/docs/deploy-minio-on-kubernetes.html#minio-distributed-server-deployment)
|
||||
* [Deployment based on MinIO Helm Chart](https://github.com/helm/charts/tree/master/stable/minio)
|
||||
|
||||
## **3. Configure Hadoop, Spark, Hive to use MinIO**
|
||||
|
||||
After successful installation navigate to the Ambari UI `http://<ambari-server>:8080/` and login using the default credentials: [**_username: admin, password: admin_**]
|
||||
|
||||

|
||||
|
||||
### **3.1 Configure Hadoop**
|
||||
|
||||
Navigate to **Services** -> **HDFS** -> **CONFIGS** -> **ADVANCED** as shown below
|
||||
|
||||

|
||||
|
||||
Navigate to **Custom core-site** to configure MinIO parameters for `_s3a_` connector
|
||||
|
||||

|
||||
|
||||
Add the following optimal entries for _core-site.xml_ to configure _s3a_ with **MinIO**. Most important options here are
|
||||
|
||||
* _fs.s3a.access.key=minio_ (Access Key to access MinIO instance, this is obtained after the deployment on k8s)
|
||||
* _fs.s3a.secret.key=minio123_ (Secret Key to access MinIO instance, this is obtained after the deployment on k8s)
|
||||
* _fs.s3a.endpoint=`http://minio-address/`_
|
||||
* _fs.s3a.path.style.acces=true_
|
||||
|
||||

|
||||
|
||||
The rest of the other optimization options are discussed in the links below
|
||||
|
||||
* [https://hadoop.apache.org/docs/current/hadoop-aws/tools/hadoop-aws/index.html](https://hadoop.apache.org/docs/current/hadoop-aws/tools/hadoop-aws/index.html)
|
||||
* [https://hadoop.apache.org/docs/r3.1.1/hadoop-aws/tools/hadoop-aws/committers.html](https://hadoop.apache.org/docs/r3.1.1/hadoop-aws/tools/hadoop-aws/committers.html)
|
||||
|
||||
Once the config changes are applied, proceed to restart **Hadoop** services.
|
||||
|
||||

|
||||
|
||||
### **3.2 Configure Spark2**
|
||||
|
||||
Navigate to **Services** -> **Spark2** -> **CONFIGS** as shown below
|
||||
|
||||

|
||||
|
||||
Navigate to “**Custom spark-defaults**” to configure MinIO parameters for `_s3a_` connector
|
||||
|
||||

|
||||
|
||||
Add the following optimal entries for _spark-defaults.conf_ to configure Spark with **MinIO**.
|
||||
|
||||
* _spark.hadoop.fs.s3a.committer.magic.enabled=true_
|
||||
* _spark.hadoop.fs.s3a.committer.name=magic_
|
||||
* _spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem_
|
||||
* _spark.hadoop.fs.s3a.path.style.access=true_
|
||||
* _spark.hadoop.mapreduce.outputcommitter.factory.scheme.s3a=org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory_
|
||||
|
||||

|
||||
|
||||
Once the config changes are applied, proceed to restart **Spark** services.
|
||||
|
||||

|
||||
|
||||
### **3.3 Configure Hive**
|
||||
|
||||
Navigate to **Services** -> **Hive** -> **CONFIGS**-> **ADVANCED** as shown below
|
||||
|
||||

|
||||
|
||||
Navigate to “**Custom hive-site**” to configure MinIO parameters for `_s3a_` connector
|
||||
|
||||

|
||||
|
||||
Add the following optimal entries for `hive-site.xml` to configure Hive with **MinIO**.
|
||||
|
||||
* _hive.blobstore.use.blobstore.as.scratchdir=true_
|
||||
* _hive.exec.input.listing.max.threads=50_
|
||||
* _hive.load.dynamic.partitions.thread=25_
|
||||
* _hive.metastore.fshandler.threads=50_
|
||||
* _hive.mv.files.threads=40_
|
||||
* _mapreduce.input.fileinputformat.list-status.num-threads=50_
|
||||
|
||||
For more information about these options please visit [https://www.cloudera.com/documentation/enterprise/5-11-x/topics/admin_hive_on_s3_tuning.html](https://www.cloudera.com/documentation/enterprise/5-11-x/topics/admin_hive_on_s3_tuning.html)
|
||||
|
||||

|
||||
|
||||
Once the config changes are applied, proceed to restart all Hive services.
|
||||
|
||||

|
||||
|
||||
## **4. Run Sample Applications**
|
||||
|
||||
After installing Hive, Hadoop and Spark successfully, we can now proceed to run some sample applications to see if they are configured appropriately. We can use Spark Pi and Spark WordCount programs to validate our Spark installation. We can also explore how to run Spark jobs from the command line and Spark shell.
|
||||
|
||||
### **4.1 Spark Pi**
|
||||
|
||||
Test the Spark installation by running the following compute intensive example, which calculates pi by “throwing darts” at a circle. The program generates points in the unit square ((0,0) to (1,1)) and counts how many points fall within the unit circle within the square. The result approximates pi.
|
||||
|
||||
Follow these steps to run the Spark Pi example:
|
||||
|
||||
* Login as user **‘spark’**.
|
||||
* When the job runs, the library can now use **MinIO** during intermediate processing.
|
||||
* Navigate to a node with the Spark client and access the spark2-client directory:
|
||||
|
||||
```
|
||||
cd /usr/hdp/current/spark2-client
|
||||
su spark
|
||||
```
|
||||
|
||||
* Run the Apache Spark Pi job in yarn-client mode, using code from **org.apache.spark**:
|
||||
|
||||
```
|
||||
./bin/spark-submit --class org.apache.spark.examples.SparkPi \
|
||||
--master yarn-client \
|
||||
--num-executors 1 \
|
||||
--driver-memory 512m \
|
||||
--executor-memory 512m \
|
||||
--executor-cores 1 \
|
||||
examples/jars/spark-examples*.jar 10
|
||||
```
|
||||
|
||||
The job should produce an output as shown below. Note the value of pi in the output.
|
||||
|
||||
```
|
||||
17/03/22 23:21:10 INFO DAGScheduler: Job 0 finished: reduce at SparkPi.scala:38, took 1.302805 s
|
||||
Pi is roughly 3.1445191445191445
|
||||
```
|
||||
|
||||
Job status can also be viewed in a browser by navigating to the YARN ResourceManager Web UI and clicking on job history server information.
|
||||
|
||||
### **4.2 WordCount**
|
||||
|
||||
WordCount is a simple program that counts how often a word occurs in a text file. The code builds a dataset of (String, Int) pairs called counts, and saves the dataset to a file.
|
||||
|
||||
The following example submits WordCount code to the Scala shell. Select an input file for the Spark WordCount example. We can use any text file as input.
|
||||
|
||||
* Login as user **‘spark’**.
|
||||
* When the job runs, the library can now use **MinIO** during intermediate processing.
|
||||
* Navigate to a node with Spark client and access the spark2-client directory:
|
||||
|
||||
```
|
||||
cd /usr/hdp/current/spark2-client
|
||||
su spark
|
||||
```
|
||||
|
||||
The following example uses _log4j.properties_ as the input file:
|
||||
|
||||
#### **4.2.1 Upload the input file to HDFS:**
|
||||
|
||||
```
|
||||
hadoop fs -copyFromLocal /etc/hadoop/conf/log4j.properties
|
||||
s3a://testbucket/testdata
|
||||
```
|
||||
|
||||
#### **4.2.2 Run the Spark shell:**
|
||||
|
||||
```
|
||||
./bin/spark-shell --master yarn-client --driver-memory 512m --executor-memory 512m
|
||||
```
|
||||
|
||||
The command should produce an output as shown below. (with additional status messages):
|
||||
|
||||
```
|
||||
Spark context Web UI available at http://172.26.236.247:4041
|
||||
Spark context available as 'sc' (master = yarn, app id = application_1490217230866_0002).
|
||||
Spark session available as 'spark'.
|
||||
Welcome to
|
||||
|
||||
|
||||
____ __
|
||||
/ __/__ ___ _____/ /__
|
||||
_\ \/ _ \/ _ `/ __/ '_/
|
||||
/___/ .__/\_,_/_/ /_/\_\ version 2.1.0.2.6.0.0-598
|
||||
/_/
|
||||
|
||||
Using Scala version 2.11.8 (Java HotSpot(TM) 64-Bit Server VM, Java 1.8.0_112)
|
||||
Type in expressions to have them evaluated.
|
||||
Type :help for more information.
|
||||
|
||||
scala>
|
||||
```
|
||||
|
||||
* At the _scala>_ prompt, submit the job by typing the following commands, Replace node names, file name, and file location with your values:
|
||||
|
||||
```
|
||||
scala> val file = sc.textFile("s3a://testbucket/testdata")
|
||||
file: org.apache.spark.rdd.RDD[String] = s3a://testbucket/testdata MapPartitionsRDD[1] at textFile at <console>:24
|
||||
|
||||
scala> val counts = file.flatMap(line => line.split(" ")).map(word => (word, 1)).reduceByKey(_ + _)
|
||||
counts: org.apache.spark.rdd.RDD[(String, Int)] = ShuffledRDD[4] at reduceByKey at <console>:25
|
||||
|
||||
scala> counts.saveAsTextFile("s3a://testbucket/wordcount")
|
||||
```
|
||||
|
||||
Use one of the following approaches to view job output:
|
||||
|
||||
View output in the Scala shell:
|
||||
|
||||
```
|
||||
scala> counts.count()
|
||||
364
|
||||
```
|
||||
|
||||
To view the output from MinIO exit the Scala shell. View WordCount job status:
|
||||
|
||||
```
|
||||
hadoop fs -ls s3a://testbucket/wordcount
|
||||
```
|
||||
|
||||
The output should be similar to the following:
|
||||
|
||||
```
|
||||
Found 3 items
|
||||
-rw-rw-rw- 1 spark spark 0 2019-05-04 01:36 s3a://testbucket/wordcount/_SUCCESS
|
||||
-rw-rw-rw- 1 spark spark 4956 2019-05-04 01:36 s3a://testbucket/wordcount/part-00000
|
||||
-rw-rw-rw- 1 spark spark 5616 2019-05-04 01:36 s3a://testbucket/wordcount/part-00001
|
||||
```
|
||||
|
After Width: | Height: | Size: 322 KiB |
|
After Width: | Height: | Size: 53 KiB |
|
After Width: | Height: | Size: 45 KiB |
|
After Width: | Height: | Size: 42 KiB |
|
After Width: | Height: | Size: 74 KiB |
|
After Width: | Height: | Size: 45 KiB |
|
After Width: | Height: | Size: 67 KiB |
|
After Width: | Height: | Size: 11 KiB |
|
After Width: | Height: | Size: 107 KiB |
|
After Width: | Height: | Size: 64 KiB |
|
After Width: | Height: | Size: 48 KiB |
|
After Width: | Height: | Size: 58 KiB |
|
After Width: | Height: | Size: 78 KiB |
|
After Width: | Height: | Size: 70 KiB |
@@ -931,10 +931,23 @@ The MinIO server configuration file is stored on the backend in json format. Upd
|
||||
"1": {
|
||||
"enable": true,
|
||||
"brokers": ["localhost:9092"],
|
||||
"topic": "bucketevents"
|
||||
"topic": "bucketevents",
|
||||
"queueDir": "",
|
||||
"queueLimit": 0,
|
||||
"tls": {
|
||||
"enable": false,
|
||||
"skipVerify": false,
|
||||
"clientAuth": 0
|
||||
},
|
||||
"sasl": {
|
||||
"enable": false,
|
||||
"username": "",
|
||||
"password": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
MinIO supports persistent event store. The persistent store will backup events when the kafka broker goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queueDir` field and the maximum limit of events in the queueDir in `queueLimit` field. For eg, the `queueDir` can be `/home/events` and `queueLimit` can be `1000`. By default, the `queueLimit` is set to 10000.
|
||||
|
||||
To update the configuration, use `mc admin config get` command to get the current configuration file for the minio deployment in json format, and save it locally.
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# Deploy MinIO on Chrooted Environment [](https://slack.min.io) [](https://goreportcard.com/report/minio/minio) [](https://hub.docker.com/r/minio/minio/) [](https://codecov.io/gh/minio/minio)
|
||||
# Deploy MinIO on Chrooted Environment [](https://slack.min.io) [](https://goreportcard.com/report/minio/minio) [](https://hub.docker.com/r/minio/minio/)
|
||||
|
||||
Chroot allows user based namespace isolation on many standard Linux deployments.
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# MinIO Server Config Guide [](https://slack.min.io) [](https://goreportcard.com/report/minio/minio) [](https://hub.docker.com/r/minio/minio/) [](https://codecov.io/gh/minio/minio)
|
||||
# MinIO Server Config Guide [](https://slack.min.io) [](https://goreportcard.com/report/minio/minio) [](https://hub.docker.com/r/minio/minio/)
|
||||
|
||||
## Configuration Directory
|
||||
|
||||
|
||||
@@ -64,6 +64,8 @@
|
||||
"enable": false,
|
||||
"brokers": null,
|
||||
"topic": "",
|
||||
"queueDir": "",
|
||||
"queueLimit": 0,
|
||||
"tls": {
|
||||
"enable": false,
|
||||
"skipVerify": false,
|
||||
@@ -73,7 +75,7 @@
|
||||
"enable": false,
|
||||
"username": "",
|
||||
"password": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"mqtt": {
|
||||
|
||||