Merge concurrent saves on the server, with vaults created from the web

Why: the point of a sync server is that two people can work at once. Picking
a winner was all this one could do while it could not read a vault.

What, end to end:
- A vault is created at /web or with `keeporg vault -create`: the server
  builds an empty KDBX 4 file, encrypts it with the password from the form,
  and stores it as version 1. An existing database can adopt a stored key
  with `keeporg db -id <id> -store-password`, which is verified against the
  stored file before it is accepted.
- A stale push to a `merge` vault is no longer refused. The server opens the
  stored version and the pushed one in memory, folds them together, and
  stores the re-encrypted result. Clients push as usual and pull what comes
  back. Any failure answers 409 merge_failed and the client keeps its copy
  aside, so a failed merge still loses nothing.
- At rest: vault contents stay KDBX-encrypted as always. A merge vault's
  password is kept AES-256-GCM-wrapped under a master key that lives outside
  the data directory, bound to the vault id, never returned by the API. A
  stolen data directory is ciphertext; a stolen data directory plus the
  master key is not. The server runs without a master key and simply refuses
  to create or merge vaults.

This moves the server inside the trust boundary for vaults it can merge,
which is inherent to merging and is stated plainly in the README, the
handover notes and the web form itself. Key-less strategies are unchanged and
remain the default for a database created without a password.

Proved: two clients editing the same version are merged rather than refused,
the newer edit of one entry wins, a merge failure leaves the stored vault
untouched, `merge` is refused without a stored key, the wrapped key never
appears in state.json or the API, and the web interface rejects a form
without its CSRF token, withholds its cookie over plain http, and only offers
groups the user administers. Checked by hand against the real thing: a vault
created in a browser, edited by keepassxc-cli on two sides from the same
version, merged by the server, and both entries read back out with
keepassxc-cli. make check, make test-race, android lint and both .deb builds
are clean.

internal/client/admin.go

@@ -155,3 +155,31 @@ func (a *API) ConfirmTOTP(ctx context.Context, code string) error {
 	return a.do(ctx, http.MethodPost, "/api/v1/auth/totp/confirm",
 		map[string]string{"code": code}, nil)
 }
+
+// CreateVault asks the server to create a new, empty KDBX vault encrypted
+// with this password. For the merge strategy the server keeps the password,
+// wrapped under its master key, because merging means opening the file.
+func (a *API) CreateVault(ctx context.Context, name, groupID, strategy, password string) (*Database, error) {
+	var out Database
+	return &out, a.do(ctx, http.MethodPost, "/api/v1/vaults", map[string]string{
+		"name": name, "owner_group_id": groupID,
+		"merge_strategy": strategy, "password": password,
+	}, &out)
+}
+
+// StoreVaultKey gives the server the password of an existing database, so it
+// can merge it. The server verifies the password against the stored file
+// before accepting it.
+func (a *API) StoreVaultKey(ctx context.Context, dbID, password string) error {
+	return a.do(ctx, http.MethodPost,
+		"/api/v1/databases/"+url.PathEscape(dbID)+"/key",
+		map[string]any{"password": password}, nil)
+}
+
+// ForgetVaultKey removes the stored password, which also ends any ability to
+// merge that database here.
+func (a *API) ForgetVaultKey(ctx context.Context, dbID string) error {
+	return a.do(ctx, http.MethodPost,
+		"/api/v1/databases/"+url.PathEscape(dbID)+"/key",
+		map[string]any{"forget": true}, nil)
+}

internal/client/api.go

@@ -107,6 +107,10 @@ type PushResult struct {
 	Version   int64  `json:"version"`
 	SHA256    string `json:"sha256"`
 	Unchanged bool   `json:"unchanged"`
+	// Merged is set when the server folded this push into another edit; the
+	// stored version is then neither copy and must be pulled.
+	Merged       bool   `json:"merged"`
+	MergeSummary string `json:"merge_summary"`
 }
 
 // Login exchanges a password and optional second factor for a session.

internal/client/decide.go

@@ -18,6 +18,9 @@ const (
 	// actManual writes the server copy beside the local file and waits for a
 	// human.
 	actManual
+	// actPushForMerge sends the local file to a server that will fold the
+	// two copies together and hand back the result.
+	actPushForMerge
 )
 
 func (a action) String() string {
@@ -34,6 +37,8 @@ func (a action) String() string {
 		return "restore"
 	case actManual:
 		return "manual-conflict"
+	case actPushForMerge:
+		return "push-for-merge"
 	}
 	return "nothing"
 }
@@ -82,6 +87,10 @@ func decide(s situation) action {
 		switch {
 		case !s.CanWrite:
 			return actRestore
+		case s.Strategy == "merge":
+			// The server holds this vault's key and merges the two copies,
+			// so the right move is to send ours and take what comes back.
+			return actPushForMerge
 		case s.Strategy == "client-wins":
 			return actForcePush
 		case s.Strategy == "manual":

internal/client/decide_test.go

@@ -31,6 +31,12 @@ func TestDecideTable(t *testing.T) {
 		{"both changed, client wins", with(func(s *situation) {
 			s.LocalSHA, s.RemoteSHA, s.RemoteVersion, s.Strategy = "b", "c", 3, "client-wins"
 		}), actForcePush},
+		{"both changed, merge", with(func(s *situation) {
+			s.LocalSHA, s.RemoteSHA, s.RemoteVersion, s.Strategy = "b", "c", 3, "merge"
+		}), actPushForMerge},
+		{"local save on a merge vault with no remote change", with(func(s *situation) {
+			s.LocalSHA, s.Strategy = "b", "merge"
+		}), actPush},
 		{"both changed, manual", with(func(s *situation) {
 			s.LocalSHA, s.RemoteSHA, s.RemoteVersion, s.Strategy = "b", "c", 3, "manual"
 		}), actManual},

internal/client/mirror.go

@@ -234,7 +234,7 @@ func (m *Mirror) reconcile(ctx context.Context, db Database) error {
 	case actPull:
 		return m.pull(ctx, db, st, path)
 
-	case actPush:
+	case actPush, actPushForMerge:
 		return m.push(ctx, db, st, path, local, false)
 
 	case actForcePush:
@@ -297,6 +297,21 @@ func (m *Mirror) push(ctx context.Context, db Database, st *fileState, path stri
 	switch {
 	case err == nil:
 		st.Version, st.SHA256 = res.Version, res.SHA256
+		if res.Merged {
+			// The server folded our save into someone else's. What it holds
+			// now is neither copy, so take it.
+			m.log.Printf("%s: server merged version %d (%s)", db.Name, res.Version, res.MergeSummary)
+			if err := m.pull(ctx, db, st, path); err != nil {
+				return err
+			}
+			m.setStatus(db, "merged", res.MergeSummary)
+			return nil
+		}
+		if res.SHA256 != digest(data) {
+			// Whatever the server stored differs from what we sent, so the
+			// local file is no longer what the vault holds.
+			return m.pull(ctx, db, st, path)
+		}
 		if !res.Unchanged {
 			m.log.Printf("%s: pushed version %d", db.Name, res.Version)
 		}
@@ -315,6 +330,12 @@ func (m *Mirror) push(ctx context.Context, db Database, st *fileState, path stri
 		if strategy == "client-wins" && !force {
 			return m.push(ctx, db, st, path, data, true)
 		}
+		if e.Code == "merge_failed" {
+			// The server could not merge, most often because this vault's
+			// own password was changed. Keep our copy and take the server's,
+			// which is exactly what server-wins does.
+			m.log.Printf("%s: the server could not merge (%s)", db.Name, e.Message)
+		}
 		copyPath, cerr := m.keepConflictCopy(path, data)
 		if cerr != nil {
 			return cerr

internal/client/mirror_test.go

@@ -11,6 +11,7 @@ import (
 	"testing"
 	"time"
 
+	"cc.tiredsysadmin.keeporg/internal/kdbx"
 	"cc.tiredsysadmin.keeporg/internal/secret"
 	"cc.tiredsysadmin.keeporg/internal/server"
 	"cc.tiredsysadmin.keeporg/internal/store"
@@ -31,7 +32,11 @@ const password = "admin-password-1"
 // "everyone" group, and a mirror logged in as the administrator.
 func newStack(t *testing.T) *stack {
 	t.Helper()
-	srv, err := server.New(server.Config{DataDir: t.TempDir()}, log.New(io.Discard, "", 0))
+	srv, err := server.New(server.Config{
+		DataDir:       t.TempDir(),
+		MasterKeyFile: filepath.Join(t.TempDir(), "master.key"),
+		VaultKDF:      kdbx.KDFParams{MemoryKiB: 64, Iterations: 1, Lanes: 1},
+	}, log.New(io.Discard, "", 0))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -360,3 +365,105 @@ func TestRunAppliesAPushedUpdateWithoutWaitingForThePoll(t *testing.T) {
 		}
 	}
 }
+
+// A merge vault end to end: this client and another one both edit version 1,
+// and the mirror ends up holding both edits.
+func TestMirrorEndsUpWithBothEditsOnAMergeVault(t *testing.T) {
+	s := newStack(t)
+	ctx := context.Background()
+	vault, err := s.admin.CreateVault(ctx, "Merge Vault", "everyone", "merge", vaultTestPassword)
+	if err != nil {
+		t.Fatal(err)
+	}
+	s.sync()
+
+	file := filepath.Join(s.mir.Dir(), "Merge Vault.kdbx")
+	base, err := os.ReadFile(file)
+	if err != nil {
+		t.Fatalf("the vault was not mirrored: %v", err)
+	}
+	if _, err := kdbx.Open(base, vaultTestPassword); err != nil {
+		t.Fatalf("the mirrored file is not a readable vault: %v", err)
+	}
+
+	// Another client pushes its own edit of version 1.
+	theirs := addEntry(t, base, "From Them", "their-password")
+	if _, err := s.admin.Push(ctx, vault.ID, 1, theirs, false); err != nil {
+		t.Fatal(err)
+	}
+
+	// This client saves its edit of version 1, as KeePassXC would.
+	mine := addEntry(t, base, "From Me", "my-password")
+	if err := os.WriteFile(file, mine, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	s.sync()
+
+	merged, err := os.ReadFile(file)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := vaultTitles(t, merged)
+	if got["From Me"] != "my-password" || got["From Them"] != "their-password" {
+		t.Fatalf("the mirrored vault holds %v, want both edits", got)
+	}
+	// And the mirror is settled: another pass must change nothing.
+	before, _ := os.ReadFile(file)
+	s.sync()
+	after, _ := os.ReadFile(file)
+	if string(before) != string(after) {
+		t.Fatal("the mirror kept rewriting the file after a merge")
+	}
+	var seen bool
+	for _, st := range s.mir.Status() {
+		if st.DatabaseID != vault.ID {
+			continue
+		}
+		seen = true
+		if st.State != "merged" {
+			t.Fatalf("vault status = %q (%s)", st.State, st.Message)
+		}
+	}
+	if !seen {
+		t.Fatal("the vault is missing from the status report")
+	}
+}
+
+const vaultTestPassword = "vault-password-12"
+
+func addEntry(t *testing.T, data []byte, title, password string) []byte {
+	t.Helper()
+	db, err := kdbx.Open(data, vaultTestPassword)
+	if err != nil {
+		t.Fatal(err)
+	}
+	db.RootGroup().Nodes = append(db.RootGroup().Nodes,
+		kdbx.NewEntry(title, "user", password, "", ""))
+	out, err := db.Marshal(vaultTestPassword)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return out
+}
+
+func vaultTitles(t *testing.T, data []byte) map[string]string {
+	t.Helper()
+	db, err := kdbx.Open(data, vaultTestPassword)
+	if err != nil {
+		t.Fatal(err)
+	}
+	out := map[string]string{}
+	for _, e := range db.RootGroup().FindAll("Entry") {
+		title, pw := "", ""
+		for _, str := range e.FindAll("String") {
+			switch str.Value("Key") {
+			case "Title":
+				title = str.Value("Value")
+			case "Password":
+				pw = str.Value("Value")
+			}
+		}
+		out[title] = pw
+	}
+	return out
+}