Merge pull request #111 from unification-com/grt_migrate

Migration to paid GRT based subgraph queries and curated AdHoc pairs lists

go-ooo/Makefile

@@ -56,4 +56,10 @@ $(GORELEASER):
 	@echo "Installing goreleaser..."
 	@(cd /tmp && go install github.com/goreleaser/goreleaser@latest)
 
-.PHONY: abigen build install build-release lint test test-verbose goreleaser release snapshot
+check-updates:
+	@echo "checking for module updates"
+	@go list -u -f '{{if (and (not (or .Main .Indirect)) .Update)}}{{.Path}}: {{.Version}} -> {{.Update.Version}}{{end}}' -m all 2> /dev/null
+	@echo "run:"
+	@echo "go get github.com/user/repo to update."
+
+.PHONY: abigen build install build-release lint test test-verbose goreleaser release snapshot check-updates

go-ooo/README.md

@@ -10,7 +10,33 @@ The Go implementation of the OoO Provider application, required to be run by pro
 
 Go v1.18+ is required to compile the `go-ooo` application.
 
-## Running
+## testapp
+
+`testapp` can be used to test AdHoc DEX queries. It is useful for quickly testing AdHoc data retrieval without needing
+to deploy and run a full development network stack (Docker `devnet`, `go-ooo` application, running on-chain queries etc.)
+
+1. Build the `testapp`
+
+```bash
+make build-testapp
+```
+
+2. Populate the temp database
+
+```bash
+./build/testapp api adhoc-update --graphnetapi [GRAPHNET_API_KEY]
+```
+
+3. Run AdHoc queries
+
+```bash
+./build/testapp api adhoc WETH.USDC.AD --graphnetapi [GRAPHNET_API_KEY]
+```
+
+## go-ooo
+
+`go-ooo`, the core OoO service application receives and processes data requests. The application requires on-chain 
+communication, and a registered key. With the Docker `devnet`, this is already configured and ready to test.
 
 First, build the Go application:
 

go-ooo/config/config.go

@@ -61,7 +61,7 @@ type SubchainConfig struct {
 }
 
 type ApiKeysConfig struct {
-	Nodereal string `mapstructure:"nodereal_key"`
+	GraphNetwork string `mapstructure:"graph_network_key"`
 }
 
 type Config struct {
@@ -117,15 +117,15 @@ func DefaultConfig() *Config {
 			Level: "info",
 		},
 		Subchain: SubchainConfig{
-			EthHttpRpc:       "https://eth.althea.net",
+			EthHttpRpc:       "https://rpc.mevblocker.io",
 			PolygonHttpRpc:   "https://polygon-rpc.com",
 			BcsHttpRpc:       "https://bsc-dataseed.binance.org",
 			XdaiHttpRpc:      "https://rpc.gnosischain.com",
 			FantomHttpRpc:    "https://finchains.io/api",
 			ShibariumHttpRpc: "https://rpc.shibrpc.com",
 		},
 		ApiKeys: ApiKeysConfig{
-			Nodereal: "",
+			GraphNetwork: "",
 		},
 	}
 }

go-ooo/config/toml.go

@@ -16,7 +16,7 @@ const DefaultConfigTemplate = `# This is a TOML config file.
 
 [chain]
 # Address of the Router smart contract
-contract_address = "{{ .Chain.ContractAddress }}"
+contract_address = "{{ .Chain.ContractAddresses }}"
 
 # Network Id, e.g. 1 for mainnet etc.
 network_id = {{ .Chain.NetworkId }}
@@ -139,7 +139,7 @@ shibarium_http_rpc = "{{ .Subchain.ShibariumHttpRpc }}"
 # some DEX GraphQL APIs require keys
 
 [api_keys]
-nodereal_key = "{{ .ApiKeys.Nodereal }}"
+graph_network_key = "{{ .ApiKeys.GraphNetwork }}"
 `
 
 var configTemplate *template.Template

go-ooo/database/database.go

@@ -78,7 +78,6 @@ func (d *DB) Migrate() (err error) {
 		&models.FailedFulfilment{},
 		&models.ToBlocks{},
 		&models.SupportedPairs{},
-		&models.DexTokens{},
 		&models.DexPairs{},
 		&models.TokenContracts{},
 		&models.VersionInfo{},
@@ -89,5 +88,7 @@ func (d *DB) Migrate() (err error) {
 
 	d.MigrateV1ToV2()
 
+	d.MigrateV2ToV3()
+
 	return
 }

go-ooo/database/migrations.go

@@ -1,5 +1,7 @@
 package database
 
+import "go-ooo/database/models"
+
 /*
   Migrations
 */
@@ -9,28 +11,47 @@ package database
 func (d *DB) MigrateV0ToV1() {
 	dbVers, _ := d.getCurrentDbSchemaVersion()
 	if dbVers.CurrentVersion == 0 {
-		d.V0ToV1DeleteAdhocTokenData()
+		d.DeleteAdhocTokenData()
 		_ = d.setDbSchemaVersion(1)
 	}
 }
 
 func (d *DB) MigrateV1ToV2() {
 	dbVers, _ := d.getCurrentDbSchemaVersion()
 	if dbVers.CurrentVersion == 1 {
-		d.V1ToV2DeleteAdhocTokenData()
+		d.DeleteAdhocTokenData()
 		_ = d.setDbSchemaVersion(2)
 	}
 }
 
-// V0ToV1DeleteAdhocTokenData is used when migrating from Db v0 to v1
-func (d *DB) V0ToV1DeleteAdhocTokenData() {
-	d.Exec("DELETE FROM dex_pairs")
-	d.Exec("DELETE FROM dex_tokens")
-	d.Exec("DELETE FROM token_contracts")
+func (d *DB) MigrateV2ToV3() {
+	dbVers, _ := d.getCurrentDbSchemaVersion()
+	if dbVers.CurrentVersion == 2 {
+		d.DeleteAdhocTokenData()
+
+		if d.Migrator().HasTable("dex_tokens") {
+			d.Migrator().DropTable("dex_tokens")
+		}
+
+		d.Migrator().DropColumn(&models.DexPairs{}, "t0_dex_token_id")
+		d.Migrator().DropColumn(&models.DexPairs{}, "t1_dex_token_id")
+		d.Migrator().DropColumn(&models.DexPairs{}, "dex_name")
+
+		_ = d.setDbSchemaVersion(3)
+	}
 }
 
-// V1ToV2DeleteAdhocTokenData is used when migrating from Db v1 to v2
-// does the same as v0 to v1 migration, because module names have changed
-func (d *DB) V1ToV2DeleteAdhocTokenData() {
-	d.V0ToV1DeleteAdhocTokenData()
+// DeleteAdhocTokenData is used when migrating from Db v0 to v1
+func (d *DB) DeleteAdhocTokenData() {
+	if d.Migrator().HasTable("dex_pairs") {
+		d.Exec("DELETE FROM dex_pairs")
+	}
+
+	if d.Migrator().HasTable("dex_tokens") {
+		d.Exec("DELETE FROM dex_tokens")
+	}
+
+	if d.Migrator().HasTable("token_contracts") {
+		d.Exec("DELETE FROM token_contracts")
+	}
 }

go-ooo/database/models/dex_pairs.go

@@ -6,34 +6,41 @@ import (
 
 type DexPairs struct {
 	gorm.Model
-	DexName         string `gorm:"index:idx_dex_pair;index:idx_dex_pair_dex_name"`
+	Chain           string `gorm:"index:idx_dex_pair;index:idx_dex_pair_chain_name"`
+	Dex             string `gorm:"index:idx_dex_pair;index:idx_dex_pair_dex_name"`
 	Pair            string `gorm:"index:idx_dex_pair;index:idx_dex_pair_pair"`
-	T0DexTokenId    uint   `gorm:"index:idx_dex_pair_t0;index:idx_dex_pair_t0_t1"`
-	T1DexTokenId    uint   `gorm:"index:idx_dex_pair_t1;index:idx_dex_pair_t0_t1"`
+	T0TokenId       uint   `gorm:"index:idx_dex_pair_t0;index:idx_dex_pair_t0_t1"`
+	T1TokenId       uint   `gorm:"index:idx_dex_pair_t1;index:idx_dex_pair_t0_t1"`
 	T0Symbol        string `gorm:"index"`
 	T1Symbol        string `gorm:"index"`
 	ContractAddress string `gorm:"index"`
 	ReserveUsd      float64
+	TxCount         uint64
+	Verified        bool
 }
 
 func (DexPairs) TableName() string {
 	return "dex_pairs"
 }
 
+func (d *DexPairs) GetChain() string {
+	return d.Chain
+}
+
 func (d *DexPairs) GetDexName() string {
-	return d.DexName
+	return d.Dex
 }
 
 func (d *DexPairs) GetPair() string {
 	return d.Pair
 }
 
 func (d *DexPairs) GetT0DexTokenId() uint {
-	return d.T0DexTokenId
+	return d.T0TokenId
 }
 
 func (d *DexPairs) GetT1DexTokenId() uint {
-	return d.T1DexTokenId
+	return d.T1TokenId
 }
 
 func (d *DexPairs) GetContractAddress() string {

go-ooo/database/queries.go

@@ -3,6 +3,7 @@ package database
 import (
 	"fmt"
 	"go-ooo/database/models"
+	"time"
 )
 
 /*
@@ -87,30 +88,30 @@ func (d *DB) PairsNoLongerSupported(pairs []string) ([]models.SupportedPairs, er
   DexPairs queries
 */
 
-func (d *DB) FindByDexPairName(base string, target string, dexName string) (models.DexPairs, error) {
+func (d *DB) FindByDexPairName(base, target, chain, dexName string) ([]models.DexPairs, error) {
 	pair := fmt.Sprintf("%s-%s", base, target)
 	pairRev := fmt.Sprintf("%s-%s", target, base)
-	result := models.DexPairs{}
+	var result []models.DexPairs
 	err := d.Where(
-		"(pair = ? OR pair = ?) AND dex_name = ?", pair, pairRev, dexName,
-	).Order("reserve_usd desc").First(&result).Error
+		"(pair = ? OR pair = ?) AND chain = ? AND dex = ? AND verified = ?", pair, pairRev, chain, dexName, true,
+	).Order("reserve_usd desc").Find(&result).Error
 	return result, err
 }
 
-/*
-  DexTokens queries
-*/
-
-func (d *DB) FindByDexTokenSymbol(symbol string, dexName string) (models.DexTokens, error) {
-	result := models.DexTokens{}
-	err := d.Where("token_symbol = ? AND dex_name = ?", symbol, dexName).First(&result).Error
+func (d *DB) FindByDexChainAddress(chain, dex, contractAddress string) (models.DexPairs, error) {
+	result := models.DexPairs{}
+	err := d.Where(
+		"chain = ? AND dex = ? AND contract_address = ?", chain, dex, contractAddress,
+	).Order("reserve_usd desc").First(&result).Error
 	return result, err
 }
 
-func (d *DB) FindByDexTokenAll(symbol string, dexName string, tokenContractsId uint) (models.DexTokens, error) {
-	result := models.DexTokens{}
-	err := d.Where("token_symbol = ? AND dex_name = ? AND token_contracts_id = ?", symbol, dexName, tokenContractsId).First(&result).Error
-	return result, err
+func (d *DB) Get100PairsForDataRefresh(chain, dex string) ([]models.DexPairs, error) {
+	var res []models.DexPairs
+	duration, _ := time.ParseDuration("-6h")
+	qTime := time.Now().Add(duration)
+	err := d.Where("chain = ? AND dex = ? AND updated_at <= ? AND verified = ?", chain, dex, qTime, true).Limit(100).Find(&res).Error
+	return res, err
 }
 
 /*
@@ -123,6 +124,12 @@ func (d *DB) FindByTokenAndAddress(symbol string, address string) (models.TokenC
 	return result, err
 }
 
+func (d *DB) FindByChainAndAddress(chain string, address string) (models.TokenContracts, error) {
+	result := models.TokenContracts{}
+	err := d.Where("chain = ? AND contract_address = ?", chain, address).First(&result).Error
+	return result, err
+}
+
 func (d *DB) FindByTokenAll(symbol string, address string, chain string) (models.TokenContracts, error) {
 	result := models.TokenContracts{}
 	err := d.Where("token_symbol = ? AND contract_address = ? AND chain = ?", symbol, address, chain).First(&result).Error