Browse Source

Refactor the codebase

main
Quentin 9 months ago
parent
commit
15e4d10fd4
  1. 1
      .gitignore
  2. 28
      auth_basic.go
  3. 115
      auth_ldap.go
  4. 29
      auth_s3.go
  5. 77
      config.go
  6. 24
      error.go
  7. 574
      main.go
  8. 17
      middleware.go
  9. 186
      s3_file.go
  10. 66
      s3_fs.go
  11. 57
      s3_path.go
  12. 128
      s3_stat.go
  13. 28
      webdav.go

1
.gitignore

@ -0,0 +1 @@
bagage

28
auth_basic.go

@ -0,0 +1,28 @@
package main
import (
"errors"
"net/http"
)
/*
* We extract the credentials from the Basic Auth headers
* (We may think to other ways to pass credentials such as a JWT)
*/
type BasicAuthExtract struct {
OnNotFound ErrorHandler
OnCreds CredsHandler
}
func (b BasicAuthExtract) ServeHTTP(w http.ResponseWriter, r *http.Request) {
username, password, ok := r.BasicAuth()
if !ok {
b.OnNotFound.WithError(errors.New("LDAP. Missing Authentication Header")).ServeHTTP(w, r)
return
}
if username == "" || password == "" {
b.OnNotFound.WithError(errors.New("LDAP. Username or password cannot be empty")).ServeHTTP(w, r)
return
}
b.OnCreds.WithCreds(username, password).ServeHTTP(w, r)
}

115
auth_ldap.go

@ -0,0 +1,115 @@
package main
import (
"errors"
"fmt"
"net/http"
"github.com/go-ldap/ldap/v3"
)
/* Check credentials against LDAP */
type LdapPreAuth struct {
WithConfig *Config
OnWrongPassword ErrorHandler
OnFailure ErrorHandler
OnCreds CredsHandler
}
func (l LdapPreAuth) WithCreds(username, password string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// 1. Connect to the server
conn, err := ldapConnect(l.WithConfig)
if err != nil {
l.OnFailure.WithError(err).ServeHTTP(w, r)
return
}
defer conn.Close()
// 2. Authenticate with provided credentials
// @FIXME we should better check the error, it could also be due to an LDAP error
err = conn.auth(username, password)
if err != nil {
l.OnWrongPassword.WithError(err).ServeHTTP(w, r)
return
}
// 3. Fetch user's profile
profile, err := conn.profile()
if err != nil {
l.OnFailure.WithError(err).ServeHTTP(w, r)
return
}
// 4. Basic checks upon users' attributes
access_key := profile.GetAttributeValue("garage_s3_access_key")
secret_key := profile.GetAttributeValue("garage_s3_secret_key")
if access_key == "" || secret_key == "" {
err = errors.New(fmt.Sprintf("Either access key or secret key is missing in LDAP for %s", conn.userDn))
l.OnFailure.WithError(err).ServeHTTP(w, r)
return
}
// 5. Send fetched credentials to the next middleware
l.OnCreds.WithCreds(access_key, secret_key).ServeHTTP(w, r)
})
}
/**
* Private logic
*/
type ldapConnector struct {
conn *ldap.Conn
config *Config
userDn string
}
func ldapConnect(c *Config) (ldapConnector, error) {
ldapSock, err := ldap.Dial("tcp", c.LdapServer)
if err != nil {
return ldapConnector{}, err
}
return ldapConnector{
conn: ldapSock,
config: c,
}, nil
}
func (l *ldapConnector) auth(username, password string) error {
l.userDn = fmt.Sprintf("%s=%s,%s", l.config.UserNameAttr, username, l.config.UserBaseDN)
return l.conn.Bind(l.userDn, password)
}
func (l *ldapConnector) profile() (*ldap.Entry, error) {
searchRequest := ldap.NewSearchRequest(
l.userDn,
ldap.ScopeBaseObject,
ldap.NeverDerefAliases,
0,
0,
false,
"(objectClass=*)",
[]string{"garage_s3_access_key", "garage_s3_secret_key"},
nil)
sr, err := l.conn.Search(searchRequest)
if err != nil {
return nil, err
}
if len(sr.Entries) != 1 {
return nil, errors.New(fmt.Sprintf("Wrong number of LDAP entries, expected 1, got", len(sr.Entries)))
}
return sr.Entries[0], nil
}
func (l *ldapConnector) Close() {
if l.conn != nil {
l.conn.Close()
l.conn = nil
}
}

29
auth_s3.go

@ -0,0 +1,29 @@
package main
import (
"net/http"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
/* Check credentials against Minio */
type S3Auth struct {
WithConfig *Config
OnMinioClient MinioClientHandler
OnFailure ErrorHandler
}
func (s S3Auth) WithCreds(access_key, secret_key string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mc, err := minio.New(s.WithConfig.Endpoint, &minio.Options{
Creds: credentials.NewStaticV4(access_key, secret_key, ""),
Secure: s.WithConfig.UseSSL,
})
if err != nil {
s.OnFailure.WithError(err).ServeHTTP(w, r)
return
}
s.OnMinioClient.WithMC(mc).ServeHTTP(w, r)
})
}

77
config.go

@ -0,0 +1,77 @@
package main
import (
"fmt"
"os"
"reflect"
)
type Config struct {
HttpListen string `env:"BAGAGE_HTTP_LISTEN" default:":8080"`
DavPath string `env:"BAGAGE_WEBDAV_PREFIX" default:"/webdav"`
LdapServer string `env:"BAGAGE_LDAP_ENDPOINT" default:"127.0.0.1:1389"`
UserBaseDN string `env:"BAGAGE_LDAP_USER_BASE_DN" default:"ou=users,dc=deuxfleurs,dc=fr"`
UserNameAttr string `env:"BAGAGE_LDAP_USERNAME_ATTR" default:"cn"`
Endpoint string `env:"BAGAGE_S3_ENDPOINT" default:"garage.deuxfleurs.fr"`
UseSSL bool `env:"BAGAGE_S3_SSL" default:"true"`
}
func (c *Config) LoadWithDefault() *Config {
c.iter(func(t reflect.StructField, v reflect.Value) {
tag := t.Tag.Get("default")
if tag == "" {
return
} else {
setKey(v, tag)
}
})
return c
}
func (c *Config) LoadWithEnv() *Config {
c.iter(func(t reflect.StructField, v reflect.Value) {
tag := t.Tag.Get("env")
if tag == "" {
return
} else if val, ok := os.LookupEnv(tag); ok {
setKey(v, val)
}
})
return c
}
func (c *Config) String() (rep string) {
rep = "Configuration:\n"
c.iter(func(t reflect.StructField, v reflect.Value) {
rep += "\t" + t.Name + ": "
if t.Type.Kind() == reflect.Bool {
rep += fmt.Sprintf("%v", v.Bool()) + "\n"
} else {
rep += "\"" + v.String() + "\"\n"
}
})
return
}
func (c *Config) iter(cb func(t reflect.StructField, v reflect.Value)) {
t := reflect.ValueOf(c).Elem()
for i := 0; i < t.Type().NumField(); i++ {
field := t.Field(i)
typeField := t.Type().Field(i)
cb(typeField, field)
}
}
func setKey(v reflect.Value, e string) {
if v.Type().Kind() == reflect.String {
v.SetString(e)
} else if v.Type().Kind() == reflect.Bool {
v.SetBool(e == "true")
} else {
panic("Unsupported type")
}
}

24
error.go

@ -0,0 +1,24 @@
package main
import (
"net/http"
)
type NotAuthorized struct{}
func (n NotAuthorized) WithError(err error) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("WWW-Authenticate", `Basic realm="Pour accéder à Bagage, veuillez entrer vos identifiants Deuxfleurs"`)
w.WriteHeader(401)
w.Write([]byte("401 Unauthorized\n"))
})
}
type InternalError struct{}
func (i InternalError) WithError(err error) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(500)
w.Write([]byte("500 Internal Server Error\n"))
})
}

574
main.go

@ -1,567 +1,35 @@
package main
import (
"context"
"errors"
"fmt"
"io"
"io/fs"
"log"
"mime"
"net/http"
"os"
"path"
"strings"
"time"
"golang.org/x/net/webdav"
"github.com/go-ldap/ldap/v3"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
type bagageCtxKey string
const garageEntry = bagageCtxKey("garage")
type garageCtx struct {
MC *minio.Client
StatCache map[string]*GarageStat
}
func EnvOrDefault(key, def string) string {
if val, ok := os.LookupEnv(key); ok {
return val
}
return def
}
func main() {
log.Println("=== Starting Bagage ===")
HttpListen := EnvOrDefault("BAGAGE_HTTP_LISTEN", ":8080")
pathPrefix := EnvOrDefault("BAGAGE_WEBDAV_PREFIX", "/webdav")
LdapServer := EnvOrDefault("BAGAGE_LDAP_ENDPOINT", "127.0.0.1:1389")
UserBaseDN := EnvOrDefault("BAGAGE_LDAP_USER_BASE_DN", "ou=users,dc=deuxfleurs,dc=fr")
UserNameAttr := EnvOrDefault("BAGAGE_LDAP_USERNAME_ATTR", "cn")
Endpoint := EnvOrDefault("BAGAGE_S3_ENDPOINT", "garage.deuxfleurs.fr")
UseSSL := EnvOrDefault("BAGAGE_S3_SSL", "true") == "true"
srv := &webdav.Handler{
Prefix: pathPrefix,
FileSystem: NewGarageFS(),
LockSystem: webdav.NewMemLS(),
Logger: func(r *http.Request, err error) {
log.Printf("INFO: %s %s %s\n", r.RemoteAddr, r.Method, r.URL)
if err != nil {
log.Printf("ERR: %v", err)
}
},
}
http.HandleFunc(pathPrefix+"/", func(w http.ResponseWriter, r *http.Request) {
username, password, ok := r.BasicAuth()
if !ok {
NotAuthorized(w, r)
return
}
ldapSock, err := ldap.Dial("tcp", LdapServer)
if err != nil {
log.Println(err)
InternalError(w, r)
return
}
defer ldapSock.Close()
// Check credential
userDn := fmt.Sprintf("%s=%s,%s", UserNameAttr, username, UserBaseDN)
err = ldapSock.Bind(userDn, password)
if err != nil {
log.Println(err)
NotAuthorized(w, r)
return
}
// Get S3 creds garage_s3_access_key garage_s3_secret_key
searchRequest := ldap.NewSearchRequest(
userDn,
ldap.ScopeBaseObject,
ldap.NeverDerefAliases,
0,
0,
false,
"(objectClass=*)",
[]string{"garage_s3_access_key", "garage_s3_secret_key"},
nil)
sr, err := ldapSock.Search(searchRequest)
if err != nil {
log.Println(err)
InternalError(w, r)
return
}
if len(sr.Entries) != 1 {
log.Println("Wrong number of LDAP entries, expected 1, got", len(sr.Entries))
InternalError(w, r)
return
}
access_key := sr.Entries[0].GetAttributeValue("garage_s3_access_key")
secret_key := sr.Entries[0].GetAttributeValue("garage_s3_secret_key")
if access_key == "" || secret_key == "" {
log.Println("Either access key or secret key is missing in LDAP for ", userDn)
InternalError(w, r)
return
}
mc, err := minio.New(Endpoint, &minio.Options{
Creds: credentials.NewStaticV4(access_key, secret_key, ""),
Secure: UseSSL,
config := (&Config{}).LoadWithDefault().LoadWithEnv()
log.Println(config)
// Assemble components to handle WebDAV requests
http.Handle(config.DavPath+"/",
BasicAuthExtract{
OnNotFound: NotAuthorized{},
OnCreds: LdapPreAuth{
WithConfig: config,
OnWrongPassword: NotAuthorized{},
OnFailure: InternalError{},
OnCreds: S3Auth{
WithConfig: config,
OnFailure: InternalError{},
OnMinioClient: WebDav{
WithConfig: config,
},
},
},
})
if err != nil {
log.Println(err)
InternalError(w, r)
return
}
nctx := context.WithValue(r.Context(), garageEntry, garageCtx{MC: mc, StatCache: make(map[string]*GarageStat)})
srv.ServeHTTP(w, r.WithContext(nctx))
return
})
if err := http.ListenAndServe(HttpListen, nil); err != nil {
if err := http.ListenAndServe(config.HttpListen, nil); err != nil {
log.Fatalf("Error with WebDAV server: %v", err)
}
}
func NotAuthorized(w http.ResponseWriter, r *http.Request) {
w.Header().Set("WWW-Authenticate", `Basic realm="Pour accéder à Bagage, veuillez entrer vos identifiants Deuxfleurs"`)
w.WriteHeader(401)
w.Write([]byte("401 Unauthorized\n"))
}
func InternalError(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(500)
w.Write([]byte("500 Internal Server Error\n"))
}
/*
/////// Select Action
If no slash or one trailing slash
return ListBuckets
Else
obj := ListObjects
If obj.Length == 1
return GetObject
Else
return obj
*/
type GarageFS struct{}
func NewGarageFS() *GarageFS {
grg := new(GarageFS)
return grg
}
func (s *GarageFS) Mkdir(ctx context.Context, name string, perm os.FileMode) error {
return errors.New("Not implemented Mkdir")
}
func (s *GarageFS) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (webdav.File, error) {
//log.Println("Stat from GarageFS.OpenFile()", name)
NewGarageStatFromFile(ctx, name)
return NewGarageFile(ctx, name)
}
func (s *GarageFS) RemoveAll(ctx context.Context, name string) error {
return errors.New("Not implemented RemoveAll")
}
func (s *GarageFS) Rename(ctx context.Context, oldName, newName string) error {
return errors.New("Not implemented Rename")
}
func (s *GarageFS) Stat(ctx context.Context, name string) (os.FileInfo, error) {
//log.Println("Stat from GarageFS.Stat()", name)
return NewGarageStat(ctx, name)
}
type GarageFile struct {
ctx context.Context
mc *minio.Client
obj *minio.Object
objw *io.PipeWriter
donew chan error
pos int64
path S3Path
}
func NewGarageFile(ctx context.Context, path string) (webdav.File, error) {
gf := new(GarageFile)
gf.ctx = ctx
gf.pos = 0
gf.mc = ctx.Value(garageEntry).(garageCtx).MC
gf.path = NewS3Path(path)
return gf, nil
}
func (gf *GarageFile) Close() error {
err := make([]error, 0)
if gf.obj != nil {
err = append(err, gf.obj.Close())
gf.obj = nil
}
if gf.objw != nil {
// wait that minio completes its transfers in background
err = append(err, gf.objw.Close())
err = append(err, <-gf.donew)
gf.donew = nil
gf.objw = nil
}
count := 0
for _, e := range err {
if e != nil {
count++
log.Println(e)
}
}
if count > 0 {
return errors.New(fmt.Sprintf("%d errors when closing this WebDAV File. Read previous logs to know more.", count))
}
return nil
}
func (gf *GarageFile) loadObject() error {
if gf.obj == nil {
obj, err := gf.mc.GetObject(gf.ctx, gf.path.bucket, gf.path.key, minio.GetObjectOptions{})
if err != nil {
return err
}
gf.obj = obj
}
return nil
}
func (gf *GarageFile) Read(p []byte) (n int, err error) {
//if gf.Stat() & OBJECT == 0 { /* @FIXME Ideally we would check against OBJECT but we need a non OPAQUE_KEY */
// return 0, os.ErrInvalid
//}
if err := gf.loadObject(); err != nil {
return 0, err
}
return gf.obj.Read(p)
}
func (gf *GarageFile) Write(p []byte) (n int, err error) {
/*if gf.path.class != OBJECT {
return 0, os.ErrInvalid
}*/
if gf.objw == nil {
if gf.pos != 0 {
return 0, errors.New("writing with an offset is not implemented")
}
r, w := io.Pipe()
gf.donew = make(chan error, 1)
gf.objw = w
contentType := mime.TypeByExtension(path.Ext(gf.path.key))
go func() {
_, err := gf.mc.PutObject(context.Background(), gf.path.bucket, gf.path.key, r, -1, minio.PutObjectOptions{ContentType: contentType})
gf.donew <- err
}()
}
return gf.objw.Write(p)
}
func (gf *GarageFile) Seek(offset int64, whence int) (int64, error) {
if err := gf.loadObject(); err != nil {
return 0, err
}
pos, err := gf.obj.Seek(offset, whence)
gf.pos += pos
return pos, err
}
/*
ReadDir reads the contents of the directory associated with the file f and returns a slice of DirEntry values in directory order. Subsequent calls on the same file will yield later DirEntry records in the directory.
If n > 0, ReadDir returns at most n DirEntry records. In this case, if ReadDir returns an empty slice, it will return an error explaining why. At the end of a directory, the error is io.EOF.
If n <= 0, ReadDir returns all the DirEntry records remaining in the directory. When it succeeds, it returns a nil error (not io.EOF).
*/
func (gf *GarageFile) Readdir(count int) ([]fs.FileInfo, error) {
if count > 0 {
return nil, errors.New("returning a limited number of directory entry is not supported in readdir")
}
if gf.path.class == ROOT {
return gf.readDirRoot(count)
} else {
return gf.readDirChild(count)
}
}
func (gf *GarageFile) readDirRoot(count int) ([]fs.FileInfo, error) {
buckets, err := gf.mc.ListBuckets(gf.ctx)
if err != nil {
return nil, err
}
entries := make([]fs.FileInfo, 0, len(buckets))
for _, bucket := range buckets {
//log.Println("Stat from GarageFile.readDirRoot()", "/"+bucket.Name)
ngf, err := NewGarageStat(gf.ctx, "/"+bucket.Name)
if err != nil {
return nil, err
}
entries = append(entries, ngf)
}
return entries, nil
}
func (gf *GarageFile) readDirChild(count int) ([]fs.FileInfo, error) {
prefix := gf.path.key
if len(prefix) > 0 && prefix[len(prefix)-1:] != "/" {
prefix = prefix + "/"
}
objs_info := gf.mc.ListObjects(gf.ctx, gf.path.bucket, minio.ListObjectsOptions{
Prefix: prefix,
Recursive: false,
})
entries := make([]fs.FileInfo, 0)
for object := range objs_info {
if object.Err != nil {
return nil, object.Err
}
//log.Println("Stat from GarageFile.readDirChild()", path.Join("/", gf.path.bucket, object.Key))
ngf, err := NewGarageStatFromObjectInfo(gf.ctx, gf.path.bucket, object)
if err != nil {
return nil, err
}
entries = append(entries, ngf)
}
return entries, nil
}
func (gf *GarageFile) Stat() (fs.FileInfo, error) {
//log.Println("Stat from GarageFile.Stat()", gf.path.path)
return NewGarageStatFromFile(gf.ctx, gf.path.path)
}
/* Implements */
// StatObject???
type GarageStat struct {
obj minio.ObjectInfo
ctx context.Context
path S3Path
}
/*
* Stat a path
*/
func NewGarageStatFromFile(ctx context.Context, path string) (*GarageStat, error) {
cache := ctx.Value(garageEntry).(garageCtx).StatCache
// Maybe this file is already in our cache?
if entry, ok := cache[path]; ok {
return entry, nil
}
// Create a placeholder in case we are creating the object
gs := new(GarageStat)
gs.ctx = ctx
gs.path = NewS3Path(path)
if gs.path.class == OPAQUE_KEY {
gs.path.class = OBJECT // known because called from GarageFile
}
gs.obj.Key = gs.path.key
gs.obj.LastModified = time.Now()
// Maybe this file exists in garage?
err := gs.Refresh()
if err != nil && !os.IsNotExist(err) {
// There is an error and this is not a 404, report it.
return nil, err
}
cache[path] = gs
return gs, nil
}
/*
* Stat a path knowing its ObjectInfo
*/
func NewGarageStatFromObjectInfo(ctx context.Context, bucket string, obj minio.ObjectInfo) (*GarageStat, error) {
gs := new(GarageStat)
gs.path = NewTrustedS3Path(bucket, obj)
gs.obj = obj
cache := ctx.Value(garageEntry).(garageCtx).StatCache
cache[gs.path.path] = gs
return gs, nil
}
/*
* Stat a path without additional information
*/
func NewGarageStat(ctx context.Context, path string) (*GarageStat, error) {
cache := ctx.Value(garageEntry).(garageCtx).StatCache
if entry, ok := cache[path]; ok {
return entry, nil
}
gs := new(GarageStat)
gs.ctx = ctx
gs.path = NewS3Path(path)
if err := gs.Refresh(); err != nil {
return nil, err
}
if gs.path.class&OPAQUE_KEY != 0 {
return nil, errors.New("Failed to precisely determine the key type, this a logic error.")
}
cache[path] = gs
cache[gs.path.path] = gs
return gs, nil
}
func (gs *GarageStat) Refresh() error {
if gs.path.class == ROOT || gs.path.class == BUCKET {
return nil
}
mc := gs.ctx.Value(garageEntry).(garageCtx).MC
// Compute the prefix to have the desired behaviour for our stat logic
prefix := gs.path.key
if prefix[len(prefix)-1:] == "/" {
prefix = prefix[:len(prefix)-1]
}
// Get info and check if the key exists
objs_info := mc.ListObjects(gs.ctx, gs.path.bucket, minio.ListObjectsOptions{
Prefix: prefix,
Recursive: false,
})
found := false
for object := range objs_info {
if object.Err != nil {
return object.Err
}
if object.Key == prefix || object.Key == prefix+"/" {
gs.obj = object
gs.path = NewTrustedS3Path(gs.path.bucket, object)
found = true
break
}
}
if !found {
return fs.ErrNotExist
}
return nil
}
func (gs *GarageStat) Name() string {
if gs.path.class == ROOT {
return "/"
} else if gs.path.class == BUCKET {
return gs.path.bucket
} else {
return path.Base(gs.path.key)
}
}
func (gs *GarageStat) Size() int64 {
return gs.obj.Size
}
func (gs *GarageStat) Mode() fs.FileMode {
if gs.path.class == OBJECT {
return fs.ModePerm
} else {
return fs.ModeDir | fs.ModePerm
}
}
func (gs *GarageStat) ModTime() time.Time {
return gs.obj.LastModified
}
func (gs *GarageStat) IsDir() bool {
return gs.path.class != OBJECT
}
func (gs *GarageStat) Sys() interface{} {
return nil
}
type S3Class int
const (
ROOT S3Class = 1 << iota
BUCKET
COMMON_PREFIX
OBJECT
OPAQUE_KEY
KEY = COMMON_PREFIX | OBJECT | OPAQUE_KEY
)
type S3Path struct {
path string
class S3Class
bucket string
key string
}
func NewS3Path(path string) S3Path {
exploded_path := strings.SplitN(path, "/", 3)
// If there is no bucket name (eg. "/")
if len(exploded_path) < 2 || exploded_path[1] == "" {
return S3Path{path, ROOT, "", ""}
}
// If there is no key
if len(exploded_path) < 3 || exploded_path[2] == "" {
return S3Path{path, BUCKET, exploded_path[1], ""}
}
return S3Path{path, OPAQUE_KEY, exploded_path[1], exploded_path[2]}
}
func NewTrustedS3Path(bucket string, obj minio.ObjectInfo) S3Path {
cl := OBJECT
if obj.Key[len(obj.Key)-1:] == "/" {
cl = COMMON_PREFIX
}
return S3Path{
path: path.Join("/", bucket, obj.Key),
bucket: bucket,
key: obj.Key,
class: cl,
}
}

17
middleware.go

@ -0,0 +1,17 @@
package main
import (
"github.com/minio/minio-go/v7"
"net/http"
)
/* We define some interface to enable our middleware to communicate */
type ErrorHandler interface {
WithError(err error) http.Handler
}
type CredsHandler interface {
WithCreds(username, password string) http.Handler
}
type MinioClientHandler interface {
WithMC(mc *minio.Client) http.Handler
}

186
s3_file.go

@ -0,0 +1,186 @@
package main
import (
"context"
"errors"
"fmt"
"io"
"io/fs"
"log"
"mime"
"path"
"github.com/minio/minio-go/v7"
"golang.org/x/net/webdav"
)
type S3File struct {
fs *S3FS
obj *minio.Object
objw *io.PipeWriter
donew chan error
pos int64
path S3Path
}
func NewS3File(s *S3FS, path string) (webdav.File, error) {
f := new(S3File)
f.fs = s
f.pos = 0
f.path = NewS3Path(path)
return f, nil
}
func (f *S3File) Close() error {
err := make([]error, 0)
if f.obj != nil {
err = append(err, f.obj.Close())
f.obj = nil
}
if f.objw != nil {
// wait that minio completes its transfers in background
err = append(err, f.objw.Close())
err = append(err, <-f.donew)
f.donew = nil
f.objw = nil
}
count := 0
for _, e := range err {
if e != nil {
count++
log.Println(e)
}
}
if count > 0 {
return errors.New(fmt.Sprintf("%d errors when closing this WebDAV File. Read previous logs to know more.", count))
}
return nil
}
func (f *S3File) loadObject() error {
if f.obj == nil {
obj, err := f.fs.mc.GetObject(f.fs.ctx, f.path.bucket, f.path.key, minio.GetObjectOptions{})
if err != nil {
return err
}
f.obj = obj
}
return nil
}
func (f *S3File) Read(p []byte) (n int, err error) {
//if f.Stat() & OBJECT == 0 { /* @FIXME Ideally we would check against OBJECT but we need a non OPAQUE_KEY */
// return 0, os.ErrInvalid
//}
if err := f.loadObject(); err != nil {
return 0, err
}
return f.obj.Read(p)
}
func (f *S3File) Write(p []byte) (n int, err error) {
/*if f.path.class != OBJECT {
return 0, os.ErrInvalid
}*/
if f.objw == nil {
if f.pos != 0 {
return 0, errors.New("writing with an offset is not implemented")
}
r, w := io.Pipe()
f.donew = make(chan error, 1)
f.objw = w
contentType := mime.TypeByExtension(path.Ext(f.path.key))
go func() {
_, err := f.fs.mc.PutObject(context.Background(), f.path.bucket, f.path.key, r, -1, minio.PutObjectOptions{ContentType: contentType})
f.donew <- err
}()
}
return f.objw.Write(p)
}
func (f *S3File) Seek(offset int64, whence int) (int64, error) {
if err := f.loadObject(); err != nil {
return 0, err
}
pos, err := f.obj.Seek(offset, whence)
f.pos += pos
return pos, err
}
/*
ReadDir reads the contents of the directory associated with the file f and returns a slice of DirEntry values in directory order. Subsequent calls on the same file will yield later DirEntry records in the directory.
If n > 0, ReadDir returns at most n DirEntry records. In this case, if ReadDir returns an empty slice, it will return an error explaining why. At the end of a directory, the error is io.EOF.
If n <= 0, ReadDir returns all the DirEntry records remaining in the directory. When it succeeds, it returns a nil error (not io.EOF).
*/
func (f *S3File) Readdir(count int) ([]fs.FileInfo, error) {
if count > 0 {
return nil, errors.New("returning a limited number of directory entry is not supported in readdir")
}
if f.path.class == ROOT {
return f.readDirRoot(count)
} else {
return f.readDirChild(count)
}
}
func (f *S3File) readDirRoot(count int) ([]fs.FileInfo, error) {
buckets, err := f.fs.mc.ListBuckets(f.fs.ctx)
if err != nil {
return nil, err
}
entries := make([]fs.FileInfo, 0, len(buckets))
for _, bucket := range buckets {
//log.Println("Stat from GarageFile.readDirRoot()", "/"+bucket.Name)
nf, err := NewS3Stat(f.fs, "/"+bucket.Name)
if err != nil {
return nil, err
}
entries = append(entries, nf)
}
return entries, nil
}
func (f *S3File) readDirChild(count int) ([]fs.FileInfo, error) {
prefix := f.path.key
if len(prefix) > 0 && prefix[len(prefix)-1:] != "/" {
prefix = prefix + "/"
}
objs_info := f.fs.mc.ListObjects(f.fs.ctx, f.path.bucket, minio.ListObjectsOptions{
Prefix: prefix,
Recursive: false,
})
entries := make([]fs.FileInfo, 0)
for object := range objs_info {
if object.Err != nil {
return nil, object.Err
}
//log.Println("Stat from GarageFile.readDirChild()", path.Join("/", f.path.bucket, object.Key))
nf, err := NewS3StatFromObjectInfo(f.fs, f.path.bucket, object)
if err != nil {
return nil, err
}
entries = append(entries, nf)
}
return entries, nil
}
func (f *S3File) Stat() (fs.FileInfo, error) {
return NewS3Stat(f.fs, f.path.path)
}

66
s3_fs.go

@ -0,0 +1,66 @@
package main
import (
"context"
"errors"
"os"
"time"
"github.com/minio/minio-go/v7"
"golang.org/x/net/webdav"
)
/*
* S3FS lifetime is limited to a single request
* Conversely, Golang's abstraction has been thought to be shared between users
* Sharing an instance between users would be very dangerous (as we would need many checks between shared values)
*/
type S3FS struct {
cache map[string]*S3Stat
mc *minio.Client
ctx context.Context
}
func NewS3FS(mc *minio.Client) S3FS {
return S3FS{
cache: make(map[string]*S3Stat),
mc: mc,
}
}
func (s S3FS) Mkdir(ctx context.Context, name string, perm os.FileMode) error {
s.ctx = ctx
return errors.New("Not implemented Mkdir")
}
func (s S3FS) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (webdav.File, error) {
s.ctx = ctx
// If the file does not exist when opening it, we create a stub
if _, ok := s.cache[name]; !ok {
st := new(S3Stat)
st.fs = &s
st.path = NewS3Path(name)
st.path.class = OBJECT
st.obj.Key = st.path.key
st.obj.LastModified = time.Now()
s.cache[name] = st
}
return NewS3File(&s, name)
}
func (s S3FS) RemoveAll(ctx context.Context, name string) error {
s.ctx = ctx
return errors.New("Not implemented RemoveAll")
}
func (s S3FS) Rename(ctx context.Context, oldName, newName string) error {
s.ctx = ctx
return errors.New("Not implemented Rename")
}
func (s S3FS) Stat(ctx context.Context, name string) (os.FileInfo, error) {
s.ctx = ctx
return NewS3Stat(&s, name)
}

57
s3_path.go

@ -0,0 +1,57 @@
package main
import (
"path"
"strings"
"github.com/minio/minio-go/v7"
)
type S3Class int
const (
ROOT S3Class = 1 << iota
BUCKET
COMMON_PREFIX
OBJECT
OPAQUE_KEY
KEY = COMMON_PREFIX | OBJECT | OPAQUE_KEY
)
type S3Path struct {
path string
class S3Class
bucket string
key string
}
func NewS3Path(path string) S3Path {
exploded_path := strings.SplitN(path, "/", 3)
// If there is no bucket name (eg. "/")
if len(exploded_path) < 2 || exploded_path[1] == "" {
return S3Path{path, ROOT, "", ""}
}
// If there is no key
if len(exploded_path) < 3 || exploded_path[2] == "" {
return S3Path{path, BUCKET, exploded_path[1], ""}
}
return S3Path{path, OPAQUE_KEY, exploded_path[1], exploded_path[2]}
}
func NewTrustedS3Path(bucket string, obj minio.ObjectInfo) S3Path {
cl := OBJECT
if obj.Key[len(obj.Key)-1:] == "/" {
cl = COMMON_PREFIX
}
return S3Path{
path: path.Join("/", bucket, obj.Key),
bucket: bucket,
key: obj.Key,
class: cl,
}
}

128
s3_stat.go

@ -0,0 +1,128 @@
package main
import (
"errors"
"io/fs"
"path"
"time"
"github.com/minio/minio-go/v7"
)
type S3Stat struct {
fs *S3FS
obj minio.ObjectInfo
path S3Path
}
/*
* Stat a path knowing its ObjectInfo
*/
func NewS3StatFromObjectInfo(fs *S3FS, bucket string, obj minio.ObjectInfo) (*S3Stat, error) {
s := new(S3Stat)
s.path = NewTrustedS3Path(bucket, obj)
s.obj = obj
s.fs = fs
fs.cache[s.path.path] = s
return s, nil
}
/*
* Stat a path without additional information
*/
func NewS3Stat(fs *S3FS, path string) (*S3Stat, error) {
cache := fs.cache
if entry, ok := cache[path]; ok {
return entry, nil
}
s := new(S3Stat)
s.fs = fs
s.path = NewS3Path(path)
if err := s.Refresh(); err != nil {
return nil, err
}
if s.path.class&OPAQUE_KEY != 0 {
return nil, errors.New("Failed to precisely determine the key type, this a logic error.")
}
cache[path] = s
cache[s.path.path] = s
return s, nil
}
func (s *S3Stat) Refresh() error {
if s.path.class == ROOT || s.path.class == BUCKET {
return nil
}
mc := s.fs.mc
// Compute the prefix to have the desired behaviour for our stat logic
prefix := s.path.key
if prefix[len(prefix)-1:] == "/" {
prefix = prefix[:len(prefix)-1]
}
// Get info and check if the key exists
objs_info := mc.ListObjects(s.fs.ctx, s.path.bucket, minio.ListObjectsOptions{
Prefix: prefix,
Recursive: false,
})
found := false
for object := range objs_info {
if object.Err != nil {
return object.Err
}
if object.Key == prefix || object.Key == prefix+"/" {
s.obj = object
s.path = NewTrustedS3Path(s.path.bucket, object)
found = true
break
}
}
if !found {
return fs.ErrNotExist
}
return nil
}
func (s *S3Stat) Name() string {
if s.path.class == ROOT {
return "/"
} else if s.path.class == BUCKET {
return s.path.bucket
} else {
return path.Base(s.path.key)
}
}
func (s *S3Stat) Size() int64 {
return s.obj.Size
}
func (s *S3Stat) Mode() fs.FileMode {
if s.path.class == OBJECT {
return fs.ModePerm
} else {
return fs.ModeDir | fs.ModePerm
}
}
func (s *S3Stat) ModTime() time.Time {
return s.obj.LastModified
}
func (s *S3Stat) IsDir() bool {
return s.path.class != OBJECT
}
func (s *S3Stat) Sys() interface{} {
return nil
}

28
webdav.go

@ -0,0 +1,28 @@
package main
import (
"github.com/minio/minio-go/v7"
"golang.org/x/net/webdav"
"log"
"net/http"
)
type WebDav struct {
WithConfig *Config
}
func (wd WebDav) WithMC(mc *minio.Client) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
(&webdav.Handler{
Prefix: wd.WithConfig.DavPath,
FileSystem: NewS3FS(mc),
LockSystem: webdav.NewMemLS(),
Logger: func(r *http.Request, err error) {
log.Printf("INFO: %s %s %s\n", r.RemoteAddr, r.Method, r.URL)
if err != nil {
log.Printf("ERR: %v", err)
}
},
}).ServeHTTP(w, r)
})
}
Loading…
Cancel
Save