初始化蓝鸟文件快递柜源码与部署文档
This commit is contained in:
lhc committed
commit
5643d22242
70 files changed
+18669
No files matched your search
@@ -0,0 +1,790 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
type testClient struct {
|
||||
a *App
|
||||
h http.Handler
|
||||
cookies map[string]*http.Cookie
|
||||
ip string
|
||||
}
|
||||
|
||||
func fixture(t *testing.T, encrypted bool) *testClient {
|
||||
t.Helper()
|
||||
c := DefaultConfig()
|
||||
c.DataDir = t.TempDir()
|
||||
c.Encryption = encrypted
|
||||
c.MinFreeBytes = 0
|
||||
c.DataBudget = 128 << 20
|
||||
c.FileBudget = 64 << 20
|
||||
a, e := Open(c)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
t.Cleanup(func() { a.Close() })
|
||||
if e = a.SetAdmin("admin", "test-password"); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
return &testClient{a: a, h: a.Handler(http.NotFoundHandler()), cookies: map[string]*http.Cookie{}, ip: "203.0.113.10"}
|
||||
}
|
||||
func (c *testClient) request(t *testing.T, method, path string, body []byte) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
r := httptest.NewRequest(method, "http://localhost:8080/api/v1"+path, bytes.NewReader(body))
|
||||
r.RemoteAddr = c.ip + ":1234"
|
||||
if method != "GET" && method != "HEAD" {
|
||||
r.Header.Set("Origin", "http://localhost:8080")
|
||||
}
|
||||
for _, cookie := range c.cookies {
|
||||
r.AddCookie(cookie)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
c.h.ServeHTTP(w, r)
|
||||
for _, cookie := range w.Result().Cookies() {
|
||||
if cookie.MaxAge < 0 {
|
||||
delete(c.cookies, cookie.Name)
|
||||
} else {
|
||||
c.cookies[cookie.Name] = cookie
|
||||
}
|
||||
}
|
||||
return w
|
||||
}
|
||||
func (c *testClient) json(t *testing.T, method, path string, v any) *httptest.ResponseRecorder {
|
||||
b, e := json.Marshal(v)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
return c.request(t, method, path, b)
|
||||
}
|
||||
func expect(t *testing.T, w *httptest.ResponseRecorder, status int) {
|
||||
t.Helper()
|
||||
if w.Code != status {
|
||||
t.Fatalf("status=%d want=%d body=%s", w.Code, status, w.Body.String())
|
||||
}
|
||||
}
|
||||
func (c *testClient) create(t *testing.T, names []string, data [][]byte, downloads int, renew bool) (string, []string) {
|
||||
t.Helper()
|
||||
fs := []ManifestFile{}
|
||||
for i, d := range data {
|
||||
fs = append(fs, ManifestFile{Name: names[i], Size: int64(len(d))})
|
||||
}
|
||||
w := c.json(t, "POST", "/uploads", Manifest{Files: fs, Days: 7, Downloads: downloads, Renew: renew})
|
||||
expect(t, w, 201)
|
||||
var result struct {
|
||||
ID string `json:"id"`
|
||||
Files []struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"files"`
|
||||
}
|
||||
if e := json.Unmarshal(w.Body.Bytes(), &result); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
ids := []string{}
|
||||
for _, f := range result.Files {
|
||||
ids = append(ids, f.ID)
|
||||
}
|
||||
return result.ID, ids
|
||||
}
|
||||
func (c *testClient) publish(t *testing.T, id string) string {
|
||||
w := c.json(t, "POST", "/uploads/"+id+"/publish", struct{}{})
|
||||
expect(t, w, 200)
|
||||
var out struct {
|
||||
Code string `json:"code"`
|
||||
}
|
||||
json.Unmarshal(w.Body.Bytes(), &out)
|
||||
if len(out.Code) != 10 {
|
||||
t.Fatal("missing code")
|
||||
}
|
||||
return out.Code
|
||||
}
|
||||
func (c *testClient) resolve(t *testing.T, code string) {
|
||||
w := c.json(t, "POST", "/pickup/resolve", map[string]string{"code": code})
|
||||
expect(t, w, 200)
|
||||
}
|
||||
func (c *testClient) upload(t *testing.T, id, fid string, b []byte) {
|
||||
expect(t, c.request(t, "PUT", "/uploads/"+id+"/files/"+fid, b), 200)
|
||||
}
|
||||
func scalar(t *testing.T, a *App, q string, args ...any) int64 {
|
||||
t.Helper()
|
||||
var n int64
|
||||
if e := a.db.QueryRow(q, args...).Scan(&n); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
return n
|
||||
}
|
||||
func TestDedupIndependentPermissionsAndCleanup(t *testing.T) {
|
||||
for _, encrypted := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "plain", true: "age"}[encrypted], func(t *testing.T) {
|
||||
c := fixture(t, encrypted)
|
||||
data := []byte(strings.Repeat("confidential-content", 1000))
|
||||
id, files := c.create(t, []string{"a.txt", "b.txt"}, [][]byte{data, data}, 3, true)
|
||||
for _, f := range files {
|
||||
c.upload(t, id, f, data)
|
||||
}
|
||||
code := c.publish(t, id)
|
||||
if n := scalar(t, c.a, "SELECT count(*) FROM objects"); n != 1 {
|
||||
t.Fatalf("dedup objects=%d", n)
|
||||
}
|
||||
var oid string
|
||||
c.a.db.QueryRow("SELECT id FROM objects").Scan(&oid)
|
||||
payload, _ := c.a.root.ReadFile(objectPath(oid))
|
||||
if encrypted && bytes.Contains(payload, data) {
|
||||
t.Fatal("plaintext at rest")
|
||||
}
|
||||
c.resolve(t, code)
|
||||
w := c.request(t, "GET", "/pickup/files/"+files[0]+"/download", nil)
|
||||
expect(t, w, 200)
|
||||
if !bytes.Equal(w.Body.Bytes(), data) {
|
||||
t.Fatal("roundtrip mismatch")
|
||||
}
|
||||
if scalar(t, c.a, "SELECT remaining FROM files WHERE id=?", files[0]) != 2 || scalar(t, c.a, "SELECT remaining FROM files WHERE id=?", files[1]) != 3 {
|
||||
t.Fatal("download counters not independent")
|
||||
}
|
||||
aExpiry := scalar(t, c.a, "SELECT expires FROM files WHERE id=?", files[0])
|
||||
bExpiry := scalar(t, c.a, "SELECT expires FROM files WHERE id=?", files[1])
|
||||
if aExpiry-bExpiry != 3*86400 {
|
||||
t.Fatalf("renewal delta=%d", aExpiry-bExpiry)
|
||||
}
|
||||
_, e := c.a.db.Exec("UPDATE files SET status='deleting' WHERE id=?", files[0])
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = c.a.Cleanup(); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if scalar(t, c.a, "SELECT count(*) FROM objects") != 1 {
|
||||
t.Fatal("deleted shared payload prematurely")
|
||||
}
|
||||
_, e = c.a.db.Exec("UPDATE files SET status='deleting' WHERE id=?", files[1])
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = c.a.Cleanup(); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if scalar(t, c.a, "SELECT count(*) FROM objects") != 0 {
|
||||
t.Fatal("unreferenced object leaked")
|
||||
}
|
||||
if _, e = c.a.root.Stat(objectPath(oid)); !os.IsNotExist(e) {
|
||||
t.Fatal("payload not removed")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
func TestLimitsAndOversizedBody(t *testing.T) {
|
||||
c := fixture(t, false)
|
||||
p := c.a.cfg.Policy
|
||||
p.MaxFiles = 1
|
||||
p.MaxFileBytes = 4
|
||||
p.MaxBatchBytes = 4
|
||||
b, _ := json.Marshal(p)
|
||||
c.a.db.Exec("UPDATE settings SET json=?", string(b))
|
||||
expect(t, c.json(t, "POST", "/uploads", Manifest{Files: []ManifestFile{{"a", 3}, {"b", 1}}}), 400)
|
||||
expect(t, c.json(t, "POST", "/uploads", Manifest{Files: []ManifestFile{{"a", 5}}}), 400)
|
||||
id, files := c.create(t, []string{"a"}, [][]byte{[]byte("1234")}, 2, false)
|
||||
expect(t, c.request(t, "PUT", "/uploads/"+id+"/files/"+files[0], []byte("12345")), 400)
|
||||
if scalar(t, c.a, "SELECT count(*) FROM objects") != 0 {
|
||||
t.Fatal("oversize object published")
|
||||
}
|
||||
expect(t, c.json(t, "POST", "/uploads/"+id+"/publish", struct{}{}), 409)
|
||||
c.upload(t, id, files[0], []byte("1234"))
|
||||
c.publish(t, id)
|
||||
var billed int64
|
||||
c.a.db.QueryRow("SELECT bytes FROM usage WHERE subject LIKE 'ip:%'").Scan(&billed)
|
||||
if billed != 8 {
|
||||
t.Fatalf("retry traffic billed=%d want=8", billed)
|
||||
}
|
||||
}
|
||||
func TestConcurrentLastDownload(t *testing.T) {
|
||||
c := fixture(t, false)
|
||||
id, files := c.create(t, []string{"a"}, [][]byte{[]byte("one")}, 1, true)
|
||||
c.upload(t, id, files[0], []byte("one"))
|
||||
c.resolve(t, c.publish(t, id))
|
||||
cookie := *c.cookies["bb_pickup"]
|
||||
var wg sync.WaitGroup
|
||||
codes := make(chan int, 12)
|
||||
for i := 0; i < 12; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
r := httptest.NewRequest("GET", "http://localhost:8080/api/v1/pickup/files/"+files[0]+"/download", nil)
|
||||
r.RemoteAddr = "203.0.113.11:1234"
|
||||
r.AddCookie(&cookie)
|
||||
w := httptest.NewRecorder()
|
||||
c.h.ServeHTTP(w, r)
|
||||
codes <- w.Code
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(codes)
|
||||
success := 0
|
||||
for n := range codes {
|
||||
if n == 200 {
|
||||
success++
|
||||
}
|
||||
}
|
||||
if success != 1 {
|
||||
t.Fatalf("accepted %d downloads", success)
|
||||
}
|
||||
if scalar(t, c.a, "SELECT remaining FROM files WHERE id=?", files[0]) != 0 {
|
||||
t.Fatal("counter below zero or unconsumed")
|
||||
}
|
||||
if scalar(t, c.a, "SELECT expires FROM files WHERE id=?", files[0]) != scalar(t, c.a, "SELECT expires FROM parcels WHERE id=?", id) {
|
||||
t.Fatal("exhausted file renewed")
|
||||
}
|
||||
}
|
||||
func TestConcurrentCapacityReservation(t *testing.T) {
|
||||
c := fixture(t, false)
|
||||
c.a.cfg.FileBudget = reserveSize(1024, false) * 2
|
||||
testRuntimeBudgets(t, c.a, c.a.cfg.FileBudget, c.a.cfg.DataBudget)
|
||||
c.a.cfg.Policy.IPDailyUploads = 100
|
||||
c.a.rateMu.Lock()
|
||||
c.a.rates["upload:"+c.ip] = &rateEntry{limiter: rate.NewLimiter(rate.Inf, 100), touched: time.Now()}
|
||||
c.a.rateMu.Unlock()
|
||||
var wg sync.WaitGroup
|
||||
statuses := make(chan int, 10)
|
||||
for i := 0; i < 10; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
b, _ := json.Marshal(Manifest{Files: []ManifestFile{{"a", 1024}}})
|
||||
r := httptest.NewRequest("POST", "http://localhost:8080/api/v1/uploads", bytes.NewReader(b))
|
||||
r.Header.Set("Origin", "http://localhost:8080")
|
||||
r.RemoteAddr = c.ip + ":12"
|
||||
w := httptest.NewRecorder()
|
||||
c.h.ServeHTTP(w, r)
|
||||
statuses <- w.Code
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(statuses)
|
||||
success := 0
|
||||
for n := range statuses {
|
||||
if n == 201 {
|
||||
success++
|
||||
} else if n != 507 {
|
||||
t.Fatalf("unexpected %d", n)
|
||||
}
|
||||
}
|
||||
if success != 2 {
|
||||
t.Fatalf("reserved %d sessions want2", success)
|
||||
}
|
||||
_, reserved, _, e := c.a.capacity()
|
||||
if e != nil || reserved != c.a.cfg.FileBudget {
|
||||
t.Fatal("reservation ledger inconsistent")
|
||||
}
|
||||
}
|
||||
func TestPasswordOwnershipAndOrigin(t *testing.T) {
|
||||
c := fixture(t, false)
|
||||
id, files := c.create(t, []string{"../../evil.html"}, [][]byte{[]byte("<script>alert(1)</script>")}, 2, false)
|
||||
other := &testClient{a: c.a, h: c.h, cookies: map[string]*http.Cookie{}, ip: "203.0.113.20"}
|
||||
expect(t, other.request(t, "PUT", "/uploads/"+id+"/files/"+files[0], []byte("abc")), 404)
|
||||
c.upload(t, id, files[0], []byte("<script>alert(1)</script>"))
|
||||
code := c.publish(t, id)
|
||||
hash, _ := hashPassword("pickup-password")
|
||||
c.a.db.Exec("UPDATE parcels SET password_hash=? WHERE id=?", hash, id)
|
||||
expect(t, c.json(t, "POST", "/pickup/resolve", map[string]string{"code": code}), 401)
|
||||
expect(t, c.json(t, "POST", "/pickup/resolve", map[string]string{"code": code, "password": "wrong"}), 404)
|
||||
expect(t, c.json(t, "POST", "/pickup/resolve", map[string]string{"code": code, "password": "pickup-password"}), 200)
|
||||
w := c.request(t, "GET", "/pickup/files/"+files[0]+"/download", nil)
|
||||
expect(t, w, 200)
|
||||
if !strings.HasPrefix(w.Header().Get("Content-Disposition"), "attachment") || w.Header().Get("Content-Type") != "application/octet-stream" {
|
||||
t.Fatal("active content served inline")
|
||||
}
|
||||
r := httptest.NewRequest("POST", "http://localhost:8080/api/v1/parcels/"+id+"/revoke", nil)
|
||||
r.Header.Set("Origin", "https://evil.example")
|
||||
r.AddCookie(c.cookies["bb_visitor"])
|
||||
w = httptest.NewRecorder()
|
||||
c.h.ServeHTTP(w, r)
|
||||
expect(t, w, 403)
|
||||
}
|
||||
func TestRenewalModesAndLifetimeCeiling(t *testing.T) {
|
||||
for _, mode := range []string{"add", "sliding"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
c := fixture(t, false)
|
||||
id, files := c.create(t, []string{"a"}, [][]byte{[]byte("content")}, 4, true)
|
||||
c.upload(t, id, files[0], []byte("content"))
|
||||
c.resolve(t, c.publish(t, id))
|
||||
expiry := scalar(t, c.a, "SELECT expires FROM files WHERE id=?", files[0])
|
||||
hard := expiry + 86400
|
||||
c.a.db.Exec("UPDATE parcels SET renewal_mode=? WHERE id=?", mode, id)
|
||||
c.a.db.Exec("UPDATE files SET hard_expires=? WHERE id=?", hard, files[0])
|
||||
expect(t, c.request(t, "GET", "/pickup/files/"+files[0]+"/download", nil), 200)
|
||||
actual := scalar(t, c.a, "SELECT expires FROM files WHERE id=?", files[0])
|
||||
expected := hard
|
||||
if mode == "sliding" {
|
||||
expected = expiry
|
||||
}
|
||||
if actual != expected {
|
||||
t.Fatalf("expiry=%d want%d", actual, expected)
|
||||
}
|
||||
if scalar(t, c.a, "SELECT remaining FROM files WHERE id=?", files[0]) != 3 {
|
||||
t.Fatal("renewal reset downloads")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type failWriter struct {
|
||||
header http.Header
|
||||
bytes int64
|
||||
}
|
||||
|
||||
func (w *failWriter) Header() http.Header { return w.header }
|
||||
func (w *failWriter) WriteHeader(int) {}
|
||||
func (w *failWriter) Write(p []byte) (int, error) { return 0, io.ErrClosedPipe }
|
||||
func TestInterruptedDownloadDoesNotRenew(t *testing.T) {
|
||||
c := fixture(t, false)
|
||||
id, files := c.create(t, []string{"a"}, [][]byte{[]byte("content")}, 3, true)
|
||||
c.upload(t, id, files[0], []byte("content"))
|
||||
c.resolve(t, c.publish(t, id))
|
||||
expiry := scalar(t, c.a, "SELECT expires FROM files WHERE id=?", files[0])
|
||||
r := httptest.NewRequest("GET", "http://localhost:8080/api/v1/pickup/files/"+files[0]+"/download", nil)
|
||||
r.RemoteAddr = c.ip + ":12"
|
||||
r.AddCookie(c.cookies["bb_pickup"])
|
||||
c.h.ServeHTTP(&failWriter{header: make(http.Header)}, r)
|
||||
if scalar(t, c.a, "SELECT expires FROM files WHERE id=?", files[0]) != expiry {
|
||||
t.Fatal("failed download renewed")
|
||||
}
|
||||
if scalar(t, c.a, "SELECT remaining FROM files WHERE id=?", files[0]) != 2 {
|
||||
t.Fatal("failed begun download not charged")
|
||||
}
|
||||
}
|
||||
func TestBackupRestoreAndMissingKey(t *testing.T) {
|
||||
c := fixture(t, true)
|
||||
id, files := c.create(t, []string{"a"}, [][]byte{[]byte("content")}, 3, true)
|
||||
c.upload(t, id, files[0], []byte("content"))
|
||||
code := c.publish(t, id)
|
||||
config := filepath.Join(t.TempDir(), "config.toml")
|
||||
os.WriteFile(config, []byte("listen='127.0.0.1:8080'\n"), 0600)
|
||||
archive := filepath.Join(t.TempDir(), "backup.tar.gz")
|
||||
if e := c.a.Backup(config, archive); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
cfg := c.a.cfg
|
||||
cfg.DataDir = t.TempDir()
|
||||
if e := Restore(cfg, archive); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
restored, e := Open(cfg)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if scalar(t, restored, "SELECT count(*) FROM objects") != 1 {
|
||||
t.Fatal("restore lost payload")
|
||||
}
|
||||
if scalar(t, restored, "SELECT count(*) FROM parcels WHERE state='restored' AND code_hash IS NULL") != 1 {
|
||||
t.Fatal("old share remains authorized")
|
||||
}
|
||||
if scalar(t, restored, "SELECT count(*) FROM parcels WHERE code_hash=?", restored.keyed(code)) != 0 {
|
||||
t.Fatal("old code restored")
|
||||
}
|
||||
restored.Close()
|
||||
os.Remove(filepath.Join(cfg.DataDir, "secrets", "age.key"))
|
||||
if a, e := Open(cfg); e == nil {
|
||||
a.Close()
|
||||
t.Fatal("missing encrypted identity silently regenerated")
|
||||
}
|
||||
}
|
||||
func TestRecoveryAndIPPersistence(t *testing.T) {
|
||||
c := fixture(t, false)
|
||||
id, files := c.create(t, []string{"a"}, [][]byte{[]byte("content")}, 3, false)
|
||||
c.upload(t, id, files[0], []byte("content"))
|
||||
cfg := c.a.cfg
|
||||
c.a.Close()
|
||||
c.a.db = nil
|
||||
c.a.root = nil
|
||||
c.a.lock = nil
|
||||
a, e := Open(cfg)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
defer a.Close()
|
||||
if scalar(t, a, "SELECT count(*) FROM parcels WHERE state='revoked'") != 1 || scalar(t, a, "SELECT count(*) FROM objects") != 0 {
|
||||
t.Fatal("partial upload became accessible or leaked")
|
||||
}
|
||||
if scalar(t, a, "SELECT uploads FROM usage WHERE subject LIKE 'ip:%'") != 1 {
|
||||
t.Fatal("IP daily counter reset")
|
||||
}
|
||||
r := httptest.NewRequest("GET", "http://localhost", nil)
|
||||
r.RemoteAddr = "203.0.113.21:80"
|
||||
r.Header.Set("X-Forwarded-For", "8.8.8.8")
|
||||
if a.remoteIP(r) != "203.0.113.21" {
|
||||
t.Fatal("untrusted forwarded IP accepted")
|
||||
}
|
||||
}
|
||||
|
||||
type gatedReader struct {
|
||||
r *bytes.Reader
|
||||
once sync.Once
|
||||
started chan<- struct{}
|
||||
release <-chan struct{}
|
||||
}
|
||||
|
||||
func (r *gatedReader) Read(p []byte) (int, error) {
|
||||
r.once.Do(func() { r.started <- struct{}{}; <-r.release })
|
||||
return r.r.Read(p)
|
||||
}
|
||||
|
||||
type sinkWriter struct {
|
||||
header http.Header
|
||||
status int
|
||||
hash hash.Hash
|
||||
count int64
|
||||
once sync.Once
|
||||
started chan<- struct{}
|
||||
release <-chan struct{}
|
||||
}
|
||||
|
||||
func (w *sinkWriter) Header() http.Header { return w.header }
|
||||
func (w *sinkWriter) WriteHeader(n int) { w.status = n }
|
||||
func (w *sinkWriter) Write(p []byte) (int, error) {
|
||||
w.once.Do(func() { w.started <- struct{}{}; <-w.release })
|
||||
if w.status == 0 {
|
||||
w.status = 200
|
||||
}
|
||||
n, e := w.hash.Write(p)
|
||||
w.count += int64(n)
|
||||
return n, e
|
||||
}
|
||||
func TestConcurrentStreamingTransfers(t *testing.T) {
|
||||
c := fixture(t, true)
|
||||
mib := 8
|
||||
if v, e := strconv.Atoi(os.Getenv("BLUEBIRD_STRESS_MIB")); e == nil && v >= 1 && v <= 100 {
|
||||
mib = v
|
||||
}
|
||||
c.a.cfg.FileBudget = int64(mib*8+64) << 20
|
||||
c.a.cfg.DataBudget = c.a.cfg.FileBudget + (64 << 20)
|
||||
testRuntimeBudgets(t, c.a, c.a.cfg.FileBudget, c.a.cfg.DataBudget)
|
||||
payload := bytes.Repeat([]byte("transfer"), mib*(1<<17))
|
||||
clients := make([]*testClient, 8)
|
||||
ids := make([]string, 8)
|
||||
files := make([]string, 8)
|
||||
for i := range clients {
|
||||
clients[i] = &testClient{a: c.a, h: c.h, cookies: map[string]*http.Cookie{}, ip: fmt.Sprintf("203.0.113.%d", 40+i)}
|
||||
id, f := clients[i].create(t, []string{"big.bin"}, [][]byte{payload}, 20, true)
|
||||
ids[i], files[i] = id, f[0]
|
||||
}
|
||||
runtime.GC()
|
||||
var before runtime.MemStats
|
||||
runtime.ReadMemStats(&before)
|
||||
started := make(chan struct{}, 8)
|
||||
release := make(chan struct{})
|
||||
var releaseOnce sync.Once
|
||||
defer releaseOnce.Do(func() { close(release) })
|
||||
var wg sync.WaitGroup
|
||||
out := make(chan int, 8)
|
||||
for i := range clients {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
r := httptest.NewRequest("PUT", "http://localhost:8080/api/v1/uploads/"+ids[i]+"/files/"+files[i], &gatedReader{r: bytes.NewReader(payload), started: started, release: release})
|
||||
r.Header.Set("Origin", "http://localhost:8080")
|
||||
r.RemoteAddr = clients[i].ip + ":1234"
|
||||
for _, cookie := range clients[i].cookies {
|
||||
r.AddCookie(cookie)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
c.h.ServeHTTP(w, r)
|
||||
out <- w.Code
|
||||
}(i)
|
||||
}
|
||||
for range 8 {
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("8 upload streams did not start together")
|
||||
}
|
||||
}
|
||||
var duringUploads runtime.MemStats
|
||||
runtime.ReadMemStats(&duringUploads)
|
||||
uploadStart := time.Now()
|
||||
releaseOnce.Do(func() { close(release) })
|
||||
wg.Wait()
|
||||
uploadDuration := time.Since(uploadStart)
|
||||
close(out)
|
||||
for status := range out {
|
||||
if status != 200 {
|
||||
t.Fatalf("upload status=%d", status)
|
||||
}
|
||||
}
|
||||
if scalar(t, c.a, "SELECT count(*) FROM objects") != 1 {
|
||||
t.Fatal("simultaneous identical content not deduplicated")
|
||||
}
|
||||
for i := range clients {
|
||||
clients[i].publish(t, ids[i])
|
||||
}
|
||||
var codeHash string
|
||||
c.a.db.QueryRow("SELECT code_hash FROM parcels WHERE id=?", ids[0]).Scan(&codeHash)
|
||||
_ = codeHash
|
||||
code := pickupCode()
|
||||
c.a.db.Exec("UPDATE parcels SET code_hash=? WHERE id=?", c.a.keyed(code), ids[0])
|
||||
clients[0].resolve(t, code)
|
||||
cookie := *clients[0].cookies["bb_pickup"]
|
||||
downloadStarted := make(chan struct{}, 16)
|
||||
downloadRelease := make(chan struct{})
|
||||
var downOnce sync.Once
|
||||
defer downOnce.Do(func() { close(downloadRelease) })
|
||||
sinks := make([]*sinkWriter, 16)
|
||||
for i := range sinks {
|
||||
sinks[i] = &sinkWriter{header: make(http.Header), hash: sha256.New(), started: downloadStarted, release: downloadRelease}
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
r := httptest.NewRequest("GET", "http://localhost:8080/api/v1/pickup/files/"+files[0]+"/download", nil)
|
||||
r.RemoteAddr = fmt.Sprintf("203.0.113.%d:1234", 80+i)
|
||||
r.AddCookie(&cookie)
|
||||
c.h.ServeHTTP(sinks[i], r)
|
||||
}(i)
|
||||
}
|
||||
for range 16 {
|
||||
select {
|
||||
case <-downloadStarted:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("16 download streams did not start together")
|
||||
}
|
||||
}
|
||||
var duringDownloads runtime.MemStats
|
||||
runtime.ReadMemStats(&duringDownloads)
|
||||
downloadStart := time.Now()
|
||||
downOnce.Do(func() { close(downloadRelease) })
|
||||
wg.Wait()
|
||||
downloadDuration := time.Since(downloadStart)
|
||||
want := sha256.Sum256(payload)
|
||||
for _, s := range sinks {
|
||||
if s.status != 200 || s.count != int64(len(payload)) || !bytes.Equal(s.hash.Sum(nil), want[:]) {
|
||||
t.Fatal("stream output checksum/count mismatch")
|
||||
}
|
||||
}
|
||||
t.Logf("age enabled; 8 simultaneous ×%dMiB uploads: %s (%.1f MiB/s); 16 simultaneous ×%dMiB downloads: %s (%.1f MiB/s); heap delta at barrier: uploads=%.2fMiB downloads=%.2fMiB; in-process HTTP handlers, excludes network/TLS", mib, uploadDuration, float64(mib*8)/uploadDuration.Seconds(), mib, downloadDuration, float64(mib*16)/downloadDuration.Seconds(), float64(int64(duringUploads.Alloc)-int64(before.Alloc))/(1<<20), float64(int64(duringDownloads.Alloc)-int64(before.Alloc))/(1<<20))
|
||||
}
|
||||
|
||||
func TestInjectedCommitFailureAndDeletionFailure(t *testing.T) {
|
||||
c := fixture(t, true)
|
||||
payload := []byte("test fault recovery")
|
||||
id, files := c.create(t, []string{"a"}, [][]byte{payload}, 3, true)
|
||||
if _, e := c.a.db.Exec("CREATE TRIGGER reject_object_attach BEFORE UPDATE OF object_id ON files WHEN NEW.object_id IS NOT NULL BEGIN SELECT RAISE(ABORT,'injected commit failure'); END"); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
expect(t, c.request(t, "PUT", "/uploads/"+id+"/files/"+files[0], payload), 500)
|
||||
if scalar(t, c.a, "SELECT count(*) FROM objects") != 0 {
|
||||
t.Fatal("rolled-back object still registered")
|
||||
}
|
||||
entries, e := os.ReadDir(filepath.Join(c.a.cfg.DataDir, "objects"))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
for _, d := range entries {
|
||||
sub, _ := os.ReadDir(filepath.Join(c.a.cfg.DataDir, "objects", d.Name()))
|
||||
if len(sub) > 0 {
|
||||
t.Fatal("failed commit left uncharged payload")
|
||||
}
|
||||
}
|
||||
c.a.db.Exec("DROP TRIGGER reject_object_attach")
|
||||
c.upload(t, id, files[0], payload)
|
||||
c.publish(t, id)
|
||||
var oid string
|
||||
c.a.db.QueryRow("SELECT id FROM objects").Scan(&oid)
|
||||
path := objectPath(oid)
|
||||
if e = c.a.root.Remove(path); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = c.a.root.Mkdir(path, 0700); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = c.a.root.WriteFile(path+"/blocker", []byte("x"), 0600); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
c.a.db.Exec("UPDATE files SET status='deleting' WHERE id=?", files[0])
|
||||
if e = c.a.Cleanup(); e == nil {
|
||||
t.Fatal("expected physical deletion failure")
|
||||
}
|
||||
if scalar(t, c.a, "SELECT count(*) FROM objects") != 1 || scalar(t, c.a, "SELECT sum(stored_bytes) FROM objects") <= 0 {
|
||||
t.Fatal("failed deletion incorrectly released capacity")
|
||||
}
|
||||
c.a.root.Remove(path + "/blocker")
|
||||
if e = c.a.Cleanup(); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if scalar(t, c.a, "SELECT count(*) FROM objects") != 0 {
|
||||
t.Fatal("retry deletion failed")
|
||||
}
|
||||
}
|
||||
func TestDeleteDuringUploadDoesNotRepublish(t *testing.T) {
|
||||
c := fixture(t, false)
|
||||
payload := []byte("delete while uploading")
|
||||
id, files := c.create(t, []string{"a"}, [][]byte{payload}, 3, false)
|
||||
started := make(chan struct{}, 1)
|
||||
release := make(chan struct{})
|
||||
defer func() {
|
||||
select {
|
||||
case <-release:
|
||||
default:
|
||||
close(release)
|
||||
}
|
||||
}()
|
||||
r := httptest.NewRequest("PUT", "http://localhost:8080/api/v1/uploads/"+id+"/files/"+files[0], &gatedReader{r: bytes.NewReader(payload), started: started, release: release})
|
||||
r.Header.Set("Origin", "http://localhost:8080")
|
||||
r.RemoteAddr = c.ip + ":12"
|
||||
r.AddCookie(c.cookies["bb_visitor"])
|
||||
done := make(chan *httptest.ResponseRecorder, 1)
|
||||
go func() { w := httptest.NewRecorder(); c.h.ServeHTTP(w, r); done <- w }()
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("stream did not start")
|
||||
}
|
||||
expect(t, c.json(t, "POST", "/admin/login", map[string]string{"username": "admin", "password": "test-password"}), 200)
|
||||
expect(t, c.json(t, "POST", "/admin/files/"+files[0]+"/delete", struct{}{}), 200)
|
||||
_, reserved, _, e := c.a.capacity()
|
||||
if e != nil || reserved != reserveSize(int64(len(payload)), false) {
|
||||
t.Fatal("active deletion released upload reservation too early")
|
||||
}
|
||||
close(release)
|
||||
expect(t, <-done, 500)
|
||||
if scalar(t, c.a, "SELECT count(*) FROM files WHERE status='ready'") != 0 {
|
||||
t.Fatal("deleted file became ready")
|
||||
}
|
||||
if scalar(t, c.a, "SELECT count(*) FROM objects") != 0 {
|
||||
t.Fatal("canceled payload leaked")
|
||||
}
|
||||
if e := c.a.Cleanup(); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
func TestAnonymousCookieResetDoesNotResetIPQuota(t *testing.T) {
|
||||
c := fixture(t, false)
|
||||
p := c.a.cfg.Policy
|
||||
p.IPDailyUploads = 1
|
||||
b, _ := json.Marshal(p)
|
||||
c.a.db.Exec("UPDATE settings SET json=?", string(b))
|
||||
c.create(t, []string{"a"}, [][]byte{[]byte("x")}, 1, false)
|
||||
fresh := &testClient{a: c.a, h: c.h, cookies: map[string]*http.Cookie{}, ip: c.ip}
|
||||
expect(t, fresh.json(t, "POST", "/uploads", Manifest{Files: []ManifestFile{{Name: "a", Size: 1}}}), 429)
|
||||
if scalar(t, c.a, "SELECT uploads FROM usage WHERE subject LIKE 'ip:%'") != 1 {
|
||||
t.Fatal("daily quota mutated on rejection")
|
||||
}
|
||||
}
|
||||
func TestFailedRestoreCannotStart(t *testing.T) {
|
||||
cfg := DefaultConfig()
|
||||
cfg.DataDir = t.TempDir()
|
||||
cfg.MinFreeBytes = 0
|
||||
archive := filepath.Join(t.TempDir(), "broken.tar.gz")
|
||||
os.WriteFile(archive, []byte("not a backup"), 0600)
|
||||
if e := Restore(cfg, archive); e == nil {
|
||||
t.Fatal("invalid archive accepted")
|
||||
}
|
||||
if a, e := Open(cfg); e == nil {
|
||||
a.Close()
|
||||
t.Fatal("partially restored service can start")
|
||||
}
|
||||
}
|
||||
func TestExclusiveLockAndZeroByteIntegrity(t *testing.T) {
|
||||
c := fixture(t, true)
|
||||
if another, e := Open(c.a.cfg); e == nil {
|
||||
another.Close()
|
||||
t.Fatal("second process lock succeeded")
|
||||
}
|
||||
id, files := c.create(t, []string{"empty"}, [][]byte{{}}, 2, true)
|
||||
c.upload(t, id, files[0], nil)
|
||||
c.resolve(t, c.publish(t, id))
|
||||
expect(t, c.request(t, "HEAD", "/pickup/files/"+files[0]+"/download", nil), 200)
|
||||
if scalar(t, c.a, "SELECT remaining FROM files WHERE id=?", files[0]) != 2 {
|
||||
t.Fatal("HEAD consumed count")
|
||||
}
|
||||
var oid string
|
||||
c.a.db.QueryRow("SELECT id FROM objects").Scan(&oid)
|
||||
path := objectPath(oid)
|
||||
raw, _ := c.a.root.ReadFile(path)
|
||||
raw[len(raw)-1] ^= 1
|
||||
c.a.root.WriteFile(path, raw, 0600)
|
||||
expect(t, c.request(t, "GET", "/pickup/files/"+files[0]+"/download", nil), 500)
|
||||
if scalar(t, c.a, "SELECT remaining FROM files WHERE id=?", files[0]) != 2 {
|
||||
t.Fatal("failed empty-content authentication consumed count")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlowUploadIdleDeadline(t *testing.T) {
|
||||
c := fixture(t, false)
|
||||
id, files := c.create(t, []string{"slow"}, [][]byte{[]byte("0123456789")}, 2, false)
|
||||
c.a.cfg.TransferIdleSeconds = 1
|
||||
c.a.cfg.MaxTransferSeconds = 10
|
||||
srv := httptest.NewServer(c.h)
|
||||
defer srv.Close()
|
||||
c.a.cfg.PublicURL = srv.URL
|
||||
conn, e := net.Dial("tcp", strings.TrimPrefix(srv.URL, "http://"))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
defer conn.Close()
|
||||
conn.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
_, e = fmt.Fprintf(conn, "PUT /api/v1/uploads/%s/files/%s HTTP/1.1\r\nHost: %s\r\nOrigin: %s\r\nCookie: bb_visitor=%s\r\nContent-Length: 10\r\nConnection: close\r\n\r\nx", id, files[0], strings.TrimPrefix(srv.URL, "http://"), srv.URL, c.cookies["bb_visitor"].Value)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
start := time.Now()
|
||||
response, e := http.ReadResponse(bufio.NewReader(conn), &http.Request{Method: "PUT"})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
io.Copy(io.Discard, response.Body)
|
||||
if response.StatusCode != 400 || time.Since(start) > 4*time.Second {
|
||||
t.Fatal("idle upload was not rejected promptly")
|
||||
}
|
||||
if scalar(t, c.a, "SELECT count(*) FROM objects") != 0 {
|
||||
t.Fatal("slow partial content became available")
|
||||
}
|
||||
}
|
||||
func TestTrustedProxyChainAndIPv6(t *testing.T) {
|
||||
c := fixture(t, false)
|
||||
_, network, _ := net.ParseCIDR("10.0.0.0/8")
|
||||
c.a.proxies = []*net.IPNet{network}
|
||||
r := httptest.NewRequest("GET", "http://localhost", nil)
|
||||
r.RemoteAddr = "10.0.0.1:80"
|
||||
r.Header.Set("X-Forwarded-For", "8.8.8.8, 203.0.113.9, 10.0.0.2")
|
||||
if c.a.remoteIP(r) != "203.0.113.9" {
|
||||
t.Fatal("trusted proxy chain accepted leftmost forged IP")
|
||||
}
|
||||
r.RemoteAddr = "[2001:db8:1:2::1]:80"
|
||||
r.Header.Del("X-Forwarded-For")
|
||||
first := c.a.remoteIP(r)
|
||||
r.RemoteAddr = "[2001:db8:1:2::ffff]:80"
|
||||
if c.a.remoteIP(r) != first {
|
||||
t.Fatal("IPv6 rotation evaded prefix grouping")
|
||||
}
|
||||
}
|
||||
|
||||
func testRuntimeBudgets(t *testing.T, a *App, file, data int64) {
|
||||
t.Helper()
|
||||
s, e := a.runtimeSettings()
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
s.FileBudget = file
|
||||
s.DataBudget = data
|
||||
b, _ := json.Marshal(s)
|
||||
if _, e = a.db.Exec("UPDATE runtime_settings SET json=? WHERE id=1", string(b)); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user