New Feature: Application Boxes in Indexer

idb/postgres/internal/encoding/encoding.go

@@ -689,6 +689,8 @@ func convertTrimmedLcAccountData(ad ledgercore.AccountData) baseAccountData {
 		TotalAssets:         ad.TotalAssets,
 		TotalAppParams:      ad.TotalAppParams,
 		TotalAppLocalStates: ad.TotalAppLocalStates,
+		TotalBoxes:          ad.TotalBoxes,
+		TotalBoxBytes:       ad.TotalBoxBytes,
 		baseOnlineAccountData: baseOnlineAccountData{
 			VoteID:          ad.VoteID,
 			SelectionID:     ad.SelectionID,
@@ -711,6 +713,8 @@ func unconvertTrimmedLcAccountData(ba baseAccountData) ledgercore.AccountData {
 			TotalAppLocalStates: ba.TotalAppLocalStates,
 			TotalAssetParams:    ba.TotalAssetParams,
 			TotalAssets:         ba.TotalAssets,
+			TotalBoxes:          ba.TotalBoxes,
+			TotalBoxBytes:       ba.TotalBoxBytes,
 		},
 		VotingData: ledgercore.VotingData{
 			VoteID:          ba.VoteID,

idb/postgres/internal/encoding/encoding_test.go

@@ -2,6 +2,7 @@ package encoding
 
 import (
 	"math/rand"
+	"reflect"
 	"testing"
 
 	"github.com/algorand/go-algorand/crypto"
@@ -580,6 +581,8 @@ func TestLcAccountDataEncoding(t *testing.T) {
 			TotalAppLocalStates: 11,
 			TotalAssetParams:    12,
 			TotalAssets:         13,
+			TotalBoxes:          20,
+			TotalBoxBytes:       21,
 		},
 		VotingData: ledgercore.VotingData{
 			VoteID:          voteID,
@@ -592,10 +595,55 @@ func TestLcAccountDataEncoding(t *testing.T) {
 	}
 	buf := EncodeTrimmedLcAccountData(ad)
 
-	expectedString := `{"onl":1,"sel":"DwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=","spend":"BgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=","stprf":"EwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==","tapl":11,"tapp":10,"tas":13,"tasp":12,"teap":9,"tsch":{"nbs":8,"nui":7},"vote":"DgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=","voteFst":16,"voteKD":18,"voteLst":17}`
+	expectedString := `{"onl":1,"sel":"DwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=","spend":"BgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=","stprf":"EwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==","tapl":11,"tapp":10,"tas":13,"tasp":12,"tbx":20,"tbxb":21,"teap":9,"tsch":{"nbs":8,"nui":7},"vote":"DgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=","voteFst":16,"voteKD":18,"voteLst":17}`
 	assert.Equal(t, expectedString, string(buf))
 
 	decodedAd, err := DecodeTrimmedLcAccountData(buf)
 	require.NoError(t, err)
 	assert.Equal(t, ad, decodedAd)
 }
+
+// structFields recursively gets all field names in a struct
+func structFields(theStruct interface{}, skip map[string]bool, names map[string]bool) {
+	rStruct := reflect.TypeOf(theStruct)
+	numFields := rStruct.NumField()
+	for i := 0; i < numFields; i++ {
+		field := rStruct.Field(i)
+		name := field.Name
+		if totalSkip, nameSkip := skip[name]; nameSkip {
+			if totalSkip {
+				continue
+			}
+		} else {
+			names[name] = true
+		}
+		if field.Type.Kind() == reflect.Struct {
+			structFields(reflect.New(field.Type).Elem().Interface(), skip, names)
+		}
+	}
+}
+
+// Test that all fields in go-algorand's AccountBaseData are either in local baseAccountData
+// or are accounted for explicitly in "skip"
+func TestBaseAccountDataVersusAccountBaseDataParity(t *testing.T) {
+
+	skip := map[string]bool{
+		"_struct":               true,
+		"MicroAlgos":            true,
+		"RewardsBase":           true,
+		"RewardedMicroAlgos":    true,
+		"MicroAlgosWithRewards": true,
+		"VotingData":            false, // skip the name, but continue with the recursion
+	}
+
+	goalNames := map[string]bool{}
+	structFields(ledgercore.AccountBaseData{}, skip, goalNames)
+	structFields(ledgercore.OnlineAccountData{}, skip, goalNames)
+
+	indexerNames := map[string]bool{}
+	structFields(baseAccountData{}, skip, indexerNames)
+
+	for name := range goalNames {
+		require.Contains(t, indexerNames, name)
+	}
+}

idb/postgres/internal/encoding/types.go

@@ -130,6 +130,8 @@ type baseAccountData struct {
 	TotalAssets         uint64             `codec:"tas"`
 	TotalAppParams      uint64             `codec:"tapp"`
 	TotalAppLocalStates uint64             `codec:"tapl"`
+	TotalBoxes          uint64             `codec:"tbx"`
+	TotalBoxBytes       uint64             `codec:"tbxb"`
 
 	baseOnlineAccountData
 }

idb/postgres/internal/schema/setup_postgres.sql

@@ -117,3 +117,11 @@ CREATE TABLE IF NOT EXISTS account_app (
 
 -- For looking up existing app local states by account
 CREATE INDEX IF NOT EXISTS account_app_by_addr_partial ON account_app(addr) WHERE NOT deleted;
+
+-- For looking up app box storage
+CREATE TABLE IF NOT EXISTS app_box (
+  app bigint NOT NULL,
+  name bytea NOT NULL,
+  value bytea NOT NULL, -- upon creation 'value' is 0x000...000 with length being the box'es size
+  PRIMARY KEY (app, name)
+);

idb/postgres/internal/writer/util.go

@@ -8,7 +8,6 @@ import (
 type copyFromChannelStruct struct {
 	ch   chan []interface{}
 	next []interface{}
-	err  error
 }
 
 func (c *copyFromChannelStruct) Next() bool {

idb/postgres/internal/writer/writer.go

@@ -9,6 +9,7 @@ import (
 	"github.com/algorand/go-algorand/data/basics"
 	"github.com/algorand/go-algorand/data/bookkeeping"
 	"github.com/algorand/go-algorand/data/transactions"
+	"github.com/algorand/go-algorand/data/transactions/logic"
 	"github.com/algorand/go-algorand/ledger/ledgercore"
 	"github.com/jackc/pgx/v4"
 
@@ -32,6 +33,8 @@ const (
 	deleteAccountAssetStmtName         = "delete_account_asset"
 	deleteAppStmtName                  = "delete_app"
 	deleteAccountAppStmtName           = "delete_account_app"
+	upsertAppBoxStmtName               = "upsert_app_box"
+	deleteAppBoxStmtName               = "delete_app_box"
 	updateAccountTotalsStmtName        = "update_account_totals"
 )
 
@@ -104,6 +107,12 @@ var statements = map[string]string{
 		(addr, app, localstate, deleted, created_at, closed_at)
 		VALUES($1, $2, 'null'::jsonb, TRUE, $3, $3) ON CONFLICT (addr, app) DO UPDATE SET
 		localstate = EXCLUDED.localstate, deleted = TRUE, closed_at = EXCLUDED.closed_at`,
+	upsertAppBoxStmtName: `INSERT INTO app_box AS ab
+		(app, name, value)
+		VALUES ($1, $2, $3)
+		ON CONFLICT (app, name) DO UPDATE SET
+		value = EXCLUDED.value`,
+	deleteAppBoxStmtName: `DELETE FROM app_box WHERE app = $1 and name = $2`,
 	updateAccountTotalsStmtName: `UPDATE metastate SET v = $1 WHERE k = '` +
 		schema.AccountTotals + `'`,
 }
@@ -122,7 +131,7 @@ func MakeWriter(tx pgx.Tx) (Writer, error) {
 	for name, query := range statements {
 		_, err := tx.Prepare(context.Background(), name, query)
 		if err != nil {
-			return Writer{}, fmt.Errorf("MakeWriter() prepare statement err: %w", err)
+			return Writer{}, fmt.Errorf("MakeWriter() prepare statement for name '%s' err: %w", name, err)
 		}
 	}
 
@@ -290,6 +299,28 @@ func writeAccountDeltas(round basics.Round, accountDeltas *ledgercore.AccountDel
 			writeAppResource(round, &appResources[i], batch)
 		}
 	}
+
+}
+
+func writeBoxMods(kvMods map[string]*string, batch *pgx.Batch) error {
+	// INSERT INTO / UPDATE / DELETE FROM `app_box`
+	// WARNING: kvMods can in theory support more general storage types than app boxes.
+	// However, here we assume that all the provided kvMods represent app boxes.
+	// If a non-box is encountered inside kvMods, an error will be returned and
+	// AddBlock() will fail with the import getting stuck at the corresponding round.
+	for key, value := range kvMods {
+		app, name, err := logic.SplitBoxKey(key)
+		if err != nil {
+			return fmt.Errorf("writeBoxMods() err: %w", err)
+		}
+		if value != nil {
+			batch.Queue(upsertAppBoxStmtName, app, []byte(name), []byte(*value))
+		} else {
+			batch.Queue(deleteAppBoxStmtName, app, []byte(name))
+		}
+	}
+
+	return nil
 }
 
 // AddBlock0 writes block 0 to the database.
@@ -309,12 +340,12 @@ func (w *Writer) AddBlock0(block *bookkeeping.Block) error {
 		_, err := results.Exec()
 		if err != nil {
 			results.Close()
-			return fmt.Errorf("AddBlock() exec err: %w", err)
+			return fmt.Errorf("AddBlock0() exec err: %w", err)
 		}
 	}
 	err := results.Close()
 	if err != nil {
-		return fmt.Errorf("AddBlock() close results err: %w", err)
+		return fmt.Errorf("AddBlock0() close results err: %w", err)
 	}
 
 	return nil
@@ -339,6 +370,12 @@ func (w *Writer) AddBlock(block *bookkeeping.Block, modifiedTxns []transactions.
 		}
 		writeAccountDeltas(block.Round(), &delta.Accts, sigTypeDeltas, &batch)
 	}
+	{
+		err := writeBoxMods(delta.KvMods, &batch)
+		if err != nil {
+			return fmt.Errorf("AddBlock() err on boxes: %w", err)
+		}
+	}
 	batch.Queue(updateAccountTotalsStmtName, encoding.EncodeAccountTotals(&delta.Totals))
 
 	results := w.tx.SendBatch(context.Background(), &batch)