aboutsummaryrefslogtreecommitdiff
path: root/internal/route
diff options
context:
space:
mode:
Diffstat (limited to 'internal/route')
-rw-r--r--internal/route/lfs/basic.go60
-rw-r--r--internal/route/lfs/basic_test.go288
-rw-r--r--internal/route/lfs/batch_test.go1
-rw-r--r--internal/route/lfs/main_test.go30
-rw-r--r--internal/route/lfs/route.go14
5 files changed, 375 insertions, 18 deletions
diff --git a/internal/route/lfs/basic.go b/internal/route/lfs/basic.go
index f0c2dc8b..626f5ff0 100644
--- a/internal/route/lfs/basic.go
+++ b/internal/route/lfs/basic.go
@@ -9,13 +9,11 @@ import (
"io"
"io/ioutil"
"net/http"
- "os"
"strconv"
"gopkg.in/macaron.v1"
log "unknwon.dev/clog/v2"
- "gogs.io/gogs/internal/conf"
"gogs.io/gogs/internal/db"
"gogs.io/gogs/internal/lfsutil"
"gogs.io/gogs/internal/strutil"
@@ -27,8 +25,25 @@ const (
basicOperationDownload = "download"
)
+type basicHandler struct {
+ // The default storage backend for uploading new objects.
+ defaultStorage lfsutil.Storage
+ // The list of available storage backends to access objects.
+ storagers map[lfsutil.Storage]lfsutil.Storager
+}
+
+// DefaultStorager returns the default storage backend.
+func (h *basicHandler) DefaultStorager() lfsutil.Storager {
+ return h.storagers[h.defaultStorage]
+}
+
+// Storager returns the given storage backend.
+func (h *basicHandler) Storager(storage lfsutil.Storage) lfsutil.Storager {
+ return h.storagers[storage]
+}
+
// GET /{owner}/{repo}.git/info/lfs/object/basic/{oid}
-func serveBasicDownload(c *macaron.Context, repo *db.Repository, oid lfsutil.OID) {
+func (h *basicHandler) serveDownload(c *macaron.Context, repo *db.Repository, oid lfsutil.OID) {
object, err := db.LFS.GetObjectByOID(repo.ID, oid)
if err != nil {
if db.IsErrLFSObjectNotExist(err) {
@@ -42,28 +57,26 @@ func serveBasicDownload(c *macaron.Context, repo *db.Repository, oid lfsutil.OID
return
}
- fpath := lfsutil.StorageLocalPath(conf.LFS.ObjectsPath, object.OID)
- r, err := os.Open(fpath)
- if err != nil {
+ s := h.Storager(object.Storage)
+ if s == nil {
internalServerError(c.Resp)
- log.Error("Failed to open object file [path: %s]: %v", fpath, err)
+ log.Error("Failed to locate the object [repo_id: %d, oid: %s]: storage %q not found", object.RepoID, object.OID, object.Storage)
return
}
- defer r.Close()
c.Header().Set("Content-Type", "application/octet-stream")
c.Header().Set("Content-Length", strconv.FormatInt(object.Size, 10))
c.Status(http.StatusOK)
- _, err = io.Copy(c.Resp, r)
+ err = s.Download(object.OID, c.Resp)
if err != nil {
- log.Error("Failed to copy object file: %v", err)
+ log.Error("Failed to download object [oid: %s]: %v", object.OID, err)
return
}
}
// PUT /{owner}/{repo}.git/info/lfs/object/basic/{oid}
-func serveBasicUpload(c *macaron.Context, repo *db.Repository, oid lfsutil.OID) {
+func (h *basicHandler) serveUpload(c *macaron.Context, repo *db.Repository, oid lfsutil.OID) {
// NOTE: LFS client will retry upload the same object if there was a partial failure,
// therefore we would like to skip ones that already exist.
_, err := db.LFS.GetObjectByOID(repo.ID, oid)
@@ -79,8 +92,25 @@ func serveBasicUpload(c *macaron.Context, repo *db.Repository, oid lfsutil.OID)
return
}
- err = db.LFS.CreateObject(repo.ID, oid, c.Req.Request.Body, lfsutil.StorageLocal)
+ s := h.DefaultStorager()
+ written, err := s.Upload(oid, c.Req.Request.Body)
+ if err != nil {
+ if err == lfsutil.ErrInvalidOID {
+ responseJSON(c.Resp, http.StatusBadRequest, responseError{
+ Message: err.Error(),
+ })
+ } else {
+ internalServerError(c.Resp)
+ log.Error("Failed to upload object [storage: %s, oid: %s]: %v", s.Storage(), oid, err)
+ }
+ return
+ }
+
+ err = db.LFS.CreateObject(repo.ID, oid, written, s.Storage())
if err != nil {
+ // NOTE: It is OK to leave the file when the whole operation failed
+ // with a DB error, a retry on client side can safely overwrite the
+ // same file as OID is seen as unique to every file.
internalServerError(c.Resp)
log.Error("Failed to create object [repo_id: %d, oid: %s]: %v", repo.ID, oid, err)
return
@@ -91,7 +121,7 @@ func serveBasicUpload(c *macaron.Context, repo *db.Repository, oid lfsutil.OID)
}
// POST /{owner}/{repo}.git/info/lfs/object/basic/verify
-func serveBasicVerify(c *macaron.Context, repo *db.Repository) {
+func (h *basicHandler) serveVerify(c *macaron.Context, repo *db.Repository) {
var request basicVerifyRequest
defer c.Req.Request.Body.Close()
err := json.NewDecoder(c.Req.Request.Body).Decode(&request)
@@ -109,7 +139,7 @@ func serveBasicVerify(c *macaron.Context, repo *db.Repository) {
return
}
- object, err := db.LFS.GetObjectByOID(repo.ID, lfsutil.OID(request.Oid))
+ object, err := db.LFS.GetObjectByOID(repo.ID, request.Oid)
if err != nil {
if db.IsErrLFSObjectNotExist(err) {
responseJSON(c.Resp, http.StatusNotFound, responseError{
@@ -123,7 +153,7 @@ func serveBasicVerify(c *macaron.Context, repo *db.Repository) {
}
if object.Size != request.Size {
- responseJSON(c.Resp, http.StatusNotFound, responseError{
+ responseJSON(c.Resp, http.StatusBadRequest, responseError{
Message: "Object size mismatch",
})
return
diff --git a/internal/route/lfs/basic_test.go b/internal/route/lfs/basic_test.go
new file mode 100644
index 00000000..db2fbe61
--- /dev/null
+++ b/internal/route/lfs/basic_test.go
@@ -0,0 +1,288 @@
+// Copyright 2020 The Gogs Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+ "bytes"
+ "io"
+ "io/ioutil"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "gopkg.in/macaron.v1"
+
+ "gogs.io/gogs/internal/db"
+ "gogs.io/gogs/internal/lfsutil"
+)
+
+var _ lfsutil.Storager = (*mockStorage)(nil)
+
+// mockStorage is a in-memory storage for LFS objects.
+type mockStorage struct {
+ buf *bytes.Buffer
+}
+
+func (s *mockStorage) Storage() lfsutil.Storage {
+ return "memory"
+}
+
+func (s *mockStorage) Upload(oid lfsutil.OID, rc io.ReadCloser) (int64, error) {
+ defer rc.Close()
+ return io.Copy(s.buf, rc)
+}
+
+func (s *mockStorage) Download(oid lfsutil.OID, w io.Writer) error {
+ _, err := io.Copy(w, s.buf)
+ return err
+}
+
+func Test_basicHandler_serveDownload(t *testing.T) {
+ s := &mockStorage{}
+ basic := &basicHandler{
+ defaultStorage: s.Storage(),
+ storagers: map[lfsutil.Storage]lfsutil.Storager{
+ s.Storage(): s,
+ },
+ }
+
+ m := macaron.New()
+ m.Use(macaron.Renderer())
+ m.Use(func(c *macaron.Context) {
+ c.Map(&db.Repository{Name: "repo"})
+ c.Map(lfsutil.OID("ef797c8118f02dfb649607dd5d3f8c7623048c9c063d532cc95c5ed7a898a64f"))
+ })
+ m.Get("/", basic.serveDownload)
+
+ tests := []struct {
+ name string
+ content string
+ mockLFSStore *db.MockLFSStore
+ expStatusCode int
+ expHeader http.Header
+ expBody string
+ }{
+ {
+ name: "object does not exist",
+ mockLFSStore: &db.MockLFSStore{
+ MockGetObjectByOID: func(repoID int64, oid lfsutil.OID) (*db.LFSObject, error) {
+ return nil, db.ErrLFSObjectNotExist{}
+ },
+ },
+ expStatusCode: http.StatusNotFound,
+ expHeader: http.Header{
+ "Content-Type": []string{"application/vnd.git-lfs+json"},
+ },
+ expBody: `{"message":"Object does not exist"}` + "\n",
+ },
+ {
+ name: "storage not found",
+ mockLFSStore: &db.MockLFSStore{
+ MockGetObjectByOID: func(repoID int64, oid lfsutil.OID) (*db.LFSObject, error) {
+ return &db.LFSObject{Storage: "bad_storage"}, nil
+ },
+ },
+ expStatusCode: http.StatusInternalServerError,
+ expHeader: http.Header{
+ "Content-Type": []string{"application/vnd.git-lfs+json"},
+ },
+ expBody: `{"message":"Internal server error"}` + "\n",
+ },
+
+ {
+ name: "object exists",
+ content: "Hello world!",
+ mockLFSStore: &db.MockLFSStore{
+ MockGetObjectByOID: func(repoID int64, oid lfsutil.OID) (*db.LFSObject, error) {
+ return &db.LFSObject{
+ Size: 12,
+ Storage: s.Storage(),
+ }, nil
+ },
+ },
+ expStatusCode: http.StatusOK,
+ expHeader: http.Header{
+ "Content-Type": []string{"application/octet-stream"},
+ "Content-Length": []string{"12"},
+ },
+ expBody: "Hello world!",
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ db.SetMockLFSStore(t, test.mockLFSStore)
+
+ s.buf = bytes.NewBufferString(test.content)
+
+ r, err := http.NewRequest("GET", "/", nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ rr := httptest.NewRecorder()
+ m.ServeHTTP(rr, r)
+
+ resp := rr.Result()
+ assert.Equal(t, test.expStatusCode, resp.StatusCode)
+ assert.Equal(t, test.expHeader, resp.Header)
+
+ body, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(t, test.expBody, string(body))
+ })
+ }
+}
+
+func Test_basicHandler_serveUpload(t *testing.T) {
+ s := &mockStorage{buf: &bytes.Buffer{}}
+ basic := &basicHandler{
+ defaultStorage: s.Storage(),
+ storagers: map[lfsutil.Storage]lfsutil.Storager{
+ s.Storage(): s,
+ },
+ }
+
+ m := macaron.New()
+ m.Use(macaron.Renderer())
+ m.Use(func(c *macaron.Context) {
+ c.Map(&db.Repository{Name: "repo"})
+ c.Map(lfsutil.OID("ef797c8118f02dfb649607dd5d3f8c7623048c9c063d532cc95c5ed7a898a64f"))
+ })
+ m.Put("/", basic.serveUpload)
+
+ tests := []struct {
+ name string
+ mockLFSStore *db.MockLFSStore
+ expStatusCode int
+ expBody string
+ }{
+ {
+ name: "object already exists",
+ mockLFSStore: &db.MockLFSStore{
+ MockGetObjectByOID: func(repoID int64, oid lfsutil.OID) (*db.LFSObject, error) {
+ return &db.LFSObject{}, nil
+ },
+ },
+ expStatusCode: http.StatusOK,
+ },
+ {
+ name: "new object",
+ mockLFSStore: &db.MockLFSStore{
+ MockGetObjectByOID: func(repoID int64, oid lfsutil.OID) (*db.LFSObject, error) {
+ return nil, db.ErrLFSObjectNotExist{}
+ },
+ MockCreateObject: func(repoID int64, oid lfsutil.OID, size int64, storage lfsutil.Storage) error {
+ return nil
+ },
+ },
+ expStatusCode: http.StatusOK,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ db.SetMockLFSStore(t, test.mockLFSStore)
+
+ r, err := http.NewRequest("PUT", "/", strings.NewReader("Hello world!"))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ rr := httptest.NewRecorder()
+ m.ServeHTTP(rr, r)
+
+ resp := rr.Result()
+ assert.Equal(t, test.expStatusCode, resp.StatusCode)
+
+ body, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(t, test.expBody, string(body))
+ })
+ }
+}
+
+func Test_basicHandler_serveVerify(t *testing.T) {
+ m := macaron.New()
+ m.Use(macaron.Renderer())
+ m.Use(func(c *macaron.Context) {
+ c.Map(&db.Repository{Name: "repo"})
+ })
+ m.Post("/", (&basicHandler{}).serveVerify)
+
+ tests := []struct {
+ name string
+ body string
+ mockLFSStore *db.MockLFSStore
+ expStatusCode int
+ expBody string
+ }{
+ {
+ name: "invalid oid",
+ body: `{"oid": "bad_oid"}`,
+ expStatusCode: http.StatusBadRequest,
+ expBody: `{"message":"Invalid oid"}` + "\n",
+ },
+ {
+ name: "object does not exist",
+ body: `{"oid":"ef797c8118f02dfb649607dd5d3f8c7623048c9c063d532cc95c5ed7a898a64f"}`,
+ mockLFSStore: &db.MockLFSStore{
+ MockGetObjectByOID: func(repoID int64, oid lfsutil.OID) (*db.LFSObject, error) {
+ return nil, db.ErrLFSObjectNotExist{}
+ },
+ },
+ expStatusCode: http.StatusNotFound,
+ expBody: `{"message":"Object does not exist"}` + "\n",
+ },
+ {
+ name: "object size mismatch",
+ body: `{"oid":"ef797c8118f02dfb649607dd5d3f8c7623048c9c063d532cc95c5ed7a898a64f"}`,
+ mockLFSStore: &db.MockLFSStore{
+ MockGetObjectByOID: func(repoID int64, oid lfsutil.OID) (*db.LFSObject, error) {
+ return &db.LFSObject{Size: 12}, nil
+ },
+ },
+ expStatusCode: http.StatusBadRequest,
+ expBody: `{"message":"Object size mismatch"}` + "\n",
+ },
+
+ {
+ name: "object exists",
+ body: `{"oid":"ef797c8118f02dfb649607dd5d3f8c7623048c9c063d532cc95c5ed7a898a64f", "size":12}`,
+ mockLFSStore: &db.MockLFSStore{
+ MockGetObjectByOID: func(repoID int64, oid lfsutil.OID) (*db.LFSObject, error) {
+ return &db.LFSObject{Size: 12}, nil
+ },
+ },
+ expStatusCode: http.StatusOK,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ db.SetMockLFSStore(t, test.mockLFSStore)
+
+ r, err := http.NewRequest("POST", "/", strings.NewReader(test.body))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ rr := httptest.NewRecorder()
+ m.ServeHTTP(rr, r)
+
+ resp := rr.Result()
+ assert.Equal(t, test.expStatusCode, resp.StatusCode)
+
+ body, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(t, test.expBody, string(body))
+ })
+ }
+}
diff --git a/internal/route/lfs/batch_test.go b/internal/route/lfs/batch_test.go
index 7dce2004..61cfb562 100644
--- a/internal/route/lfs/batch_test.go
+++ b/internal/route/lfs/batch_test.go
@@ -23,6 +23,7 @@ func Test_serveBatch(t *testing.T) {
conf.SetMockServer(t, conf.ServerOpts{
ExternalURL: "https://gogs.example.com/",
})
+
m := macaron.New()
m.Use(func(c *macaron.Context) {
c.Map(&db.User{Name: "owner"})
diff --git a/internal/route/lfs/main_test.go b/internal/route/lfs/main_test.go
new file mode 100644
index 00000000..ddc04c1b
--- /dev/null
+++ b/internal/route/lfs/main_test.go
@@ -0,0 +1,30 @@
+// Copyright 2020 The Gogs Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+ "flag"
+ "fmt"
+ "os"
+ "testing"
+
+ log "unknwon.dev/clog/v2"
+
+ "gogs.io/gogs/internal/testutil"
+)
+
+func TestMain(m *testing.M) {
+ flag.Parse()
+ if !testing.Verbose() {
+ // Remove the primary logger and register a noop logger.
+ log.Remove(log.DefaultConsoleName)
+ err := log.New("noop", testutil.InitNoopLogger)
+ if err != nil {
+ fmt.Println(err)
+ os.Exit(1)
+ }
+ }
+ os.Exit(m.Run())
+}
diff --git a/internal/route/lfs/route.go b/internal/route/lfs/route.go
index 08045ac5..16478803 100644
--- a/internal/route/lfs/route.go
+++ b/internal/route/lfs/route.go
@@ -13,6 +13,7 @@ import (
log "unknwon.dev/clog/v2"
"gogs.io/gogs/internal/authutil"
+ "gogs.io/gogs/internal/conf"
"gogs.io/gogs/internal/db"
"gogs.io/gogs/internal/lfsutil"
)
@@ -26,10 +27,17 @@ func RegisterRoutes(r *macaron.Router) {
r.Group("", func() {
r.Post("/objects/batch", authorize(db.AccessModeRead), verifyAccept, verifyContentTypeJSON, serveBatch)
r.Group("/objects/basic", func() {
+
+ basic := &basicHandler{
+ defaultStorage: lfsutil.Storage(conf.LFS.Storage),
+ storagers: map[lfsutil.Storage]lfsutil.Storager{
+ lfsutil.StorageLocal: &lfsutil.LocalStorage{Root: conf.LFS.ObjectsPath},
+ },
+ }
r.Combo("/:oid", verifyOID()).
- Get(authorize(db.AccessModeRead), serveBasicDownload).
- Put(authorize(db.AccessModeWrite), verifyContentTypeStream, serveBasicUpload)
- r.Post("/verify", authorize(db.AccessModeWrite), verifyAccept, verifyContentTypeJSON, serveBasicVerify)
+ Get(authorize(db.AccessModeRead), basic.serveDownload).
+ Put(authorize(db.AccessModeWrite), verifyContentTypeStream, basic.serveUpload)
+ r.Post("/verify", authorize(db.AccessModeWrite), verifyAccept, verifyContentTypeJSON, basic.serveVerify)
})
}, authenticate())
}
Powered by cgit, Themed by Ted Yin.