#71 Create basic wiki cyberd indexer

.gitignore

@@ -11,3 +11,4 @@ node_modules/
 
 initial-rank
 initial_guess_impact_results
+*enwiki*

cosmos/poc/testnet/config.toml

@@ -143,7 +143,7 @@ private_peer_ids = ""
 ##### mempool configuration options #####
 [mempool]
 
-recheck = false
+recheck = true
 recheck_empty = true
 broadcast = true
 wal_dir = "data/mempool.wal"

cosmos/poc/wiki/client.go

@@ -0,0 +1,134 @@
+package main
+
+import (
+	"fmt"
+	"github.com/cosmos/cosmos-sdk/client/context"
+	cli "github.com/cosmos/cosmos-sdk/client/context"
+	"github.com/cosmos/cosmos-sdk/client/keys"
+	cskeys "github.com/cosmos/cosmos-sdk/crypto/keys"
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	authcmd "github.com/cosmos/cosmos-sdk/x/auth/client/cli"
+	authtxb "github.com/cosmos/cosmos-sdk/x/auth/client/txbuilder"
+	"github.com/cybercongress/cyberd/cosmos/poc/app"
+	"github.com/cybercongress/cyberd/cosmos/poc/app/storage"
+	"github.com/cybercongress/cyberd/cosmos/poc/claim/common"
+	"github.com/spf13/viper"
+	rpcclient "github.com/tendermint/tendermint/rpc/client"
+	"os"
+	"time"
+)
+
+func InitAddLink() func([]Link) {
+
+	chainId := viper.GetString(common.FlagChainId)
+	address := viper.GetString(common.FlagAddress)
+	addr, name := accountFromAddress(address)
+
+	cdc := app.MakeCodec()
+	cliCtx := newCLIContext(name, chainId).
+		WithCodec(cdc).
+		WithAccountDecoder(authcmd.GetAccountDecoder(cdc))
+
+	accountNumber, _ := cliCtx.GetAccountNumber(addr)
+	seq, _ := cliCtx.GetAccountSequence(addr)
+
+	txCtx := authtxb.TxBuilder{
+		ChainID:       chainId,
+		Gas:           10000000,
+		AccountNumber: accountNumber,
+		Sequence:      seq,
+		Fee:           "",
+		Memo:          "",
+		Codec:         cdc,
+	}
+
+	return func(links []Link) {
+
+		msges := make([]sdk.Msg, 0, len(links))
+		for _, link := range links {
+			msges = append(msges, app.NewMsgLink(addr, storage.Cid(link.from), storage.Cid(link.to)))
+		}
+
+		sendTx(address, txCtx, cliCtx, msges)
+		txCtx.Sequence++
+	}
+}
+
+func sendTx(address string, txCtx authtxb.TxBuilder, cliCtx context.CLIContext, msgs []sdk.Msg) {
+
+	passphrase := viper.GetString(common.FlagPassphrase)
+	txBytes, err := txCtx.BuildAndSign(cliCtx.From, passphrase, msgs)
+	if err != nil {
+		panic(err)
+	}
+
+	result, err := cliCtx.BroadcastTxSync(txBytes)
+
+	if err != nil {
+		println("Error during broadcasting tx. Rebrodcasting ...")
+		println(err.Error())
+		sendTx(address, txCtx, cliCtx, msgs)
+	}
+
+	if result.Code != 0 {
+		println("Error during broadcasting tx")
+		println(string(result.Log))
+		time.Sleep(5 * time.Second)
+		addr, _ := accountFromAddress(address)
+		seq, _ := cliCtx.GetAccountSequence(addr)
+		txCtx.Sequence = seq
+		sendTx(address, txCtx, cliCtx, msgs)
+	}
+}
+
+func newCLIContext(accName string, chainId string) cli.CLIContext {
+
+	nodeUrl := viper.GetString(common.FlagNode)
+	node := rpcclient.NewHTTP(nodeUrl, "/websocket")
+	verifier := &common.NoopVerifier{ChainId: chainId}
+
+	return cli.CLIContext{
+		Client:        node,
+		NodeURI:       "",
+		AccountStore:  "acc",
+		From:          accName,
+		Height:        0,
+		TrustNode:     true,
+		UseLedger:     false,
+		Async:         false,
+		JSON:          false,
+		PrintResponse: true,
+		Verifier:      verifier,
+	}
+}
+
+func accountFromAddress(from string) (fromAddr sdk.AccAddress, fromName string) {
+	if from == "" {
+		return nil, ""
+	}
+
+	keybase, err := keys.GetKeyBase()
+	if err != nil {
+		fmt.Println("no keybase found")
+		os.Exit(1)
+	}
+
+	var info cskeys.Info
+	if addr, err := sdk.AccAddressFromBech32(from); err == nil {
+		info, err = keybase.GetByAddress(addr)
+		if err != nil {
+			fmt.Printf("could not find key %s\n", from)
+			os.Exit(1)
+		}
+	} else {
+		info, err = keybase.Get(from)
+		if err != nil {
+			fmt.Printf("could not find key %s\n", from)
+			os.Exit(1)
+		}
+	}
+
+	fromAddr = info.GetAddress()
+	fromName = info.GetName()
+	return
+}

cosmos/poc/wiki/index.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"bufio"
+	"os"
+	"regexp"
+	"strings"
+	"unicode"
+)
+
+func Index() {
+
+	startArticleId := int64(1)
+
+	sendLinks := InitAddLink()
+
+	f, err := os.OpenFile("enwiki-latest-all-titles", 0, 0)
+	if err != nil {
+		panic(err)
+	}
+	br := bufio.NewReader(f)
+	defer f.Close()
+
+	reg, err := regexp.Compile("[^a-zA-Z0-9]+")
+
+	counter := int64(0)
+	links := make([]Link, 0, 100)
+	for {
+
+		line, err := br.ReadString('\n')
+
+		if err != nil {
+			break
+		}
+
+		if counter < startArticleId {
+			counter++
+			continue
+		}
+
+		split := strings.Split(strings.TrimSuffix(line, "\n"), "\t")
+		ids := strings.Split(split[1], "_")
+
+		for _, id := range ids {
+
+			id = reg.ReplaceAllString(id, "")
+			id = strings.ToLower(id)
+
+			if len(id) == 0 || id == "" {
+				continue
+			}
+
+			if len(id) == 1 && unicode.IsSymbol(rune(id[0])) {
+				continue
+			}
+
+			page := ".wiki/wiki/" + split[1] + ".html"
+			links = append(links, Link{from: id, to: page})
+			counter++
+
+			if len(links) == 1000 {
+				println(split[1])
+				println(counter)
+				sendLinks(links)
+				links = make([]Link, 0, 100)
+			}
+		}
+	}
+}

cosmos/poc/wiki/wiki.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"fmt"
+	"github.com/cybercongress/cyberd/cosmos/poc/claim/common"
+	"github.com/mitchellh/go-homedir"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"os"
+)
+
+type Link struct {
+	from string
+	to   string
+}
+
+var (
+	wiki = &cobra.Command{
+		Use:   "wiki",
+		Short: "Use to index wiki",
+	}
+)
+
+/*
+./wiki start
+--node=127.0.0.1:34657 --passphrase=1q2w3e4r \
+--address=cosmos1g7e74lxpwlcsza8v0nca2hwahluqcv4r4d3p8p \
+--chain-id=test-chain-gRXWCL
+*/
+
+func main() {
+
+	wiki.AddCommand(StartCmd())
+	homeDir, err := getHomeDir()
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	viper.SetDefault("home", homeDir+"/.cyberdwiki")
+
+	if err := wiki.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func StartCmd() *cobra.Command {
+
+	cmd := &cobra.Command{
+		Use:   "start",
+		Short: "Start wiki indexing",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			Index()
+			return nil
+		},
+	}
+
+	cmd.Flags().String(common.FlagPassphrase, "", "Passphrase of account to claim from")
+	cmd.Flags().String(common.FlagChainId, "", "Chain Id")
+	cmd.Flags().String(common.FlagAddress, "", "ClaimFrom of account to claim from")
+	cmd.Flags().String(common.FlagNode, "127.0.0.1:26657", "Node url to connect")
+
+	viper.BindPFlag(common.FlagPassphrase, cmd.Flags().Lookup(common.FlagPassphrase))
+	viper.BindPFlag(common.FlagChainId, cmd.Flags().Lookup(common.FlagChainId))
+	viper.BindPFlag(common.FlagAddress, cmd.Flags().Lookup(common.FlagAddress))
+	viper.BindPFlag(common.FlagNode, cmd.Flags().Lookup(common.FlagNode))
+	return cmd
+}
+
+func getHomeDir() (string, error) {
+	home, err := homedir.Dir()
+	if err != nil {
+		return "", err
+	}
+	return home, nil
+}

docs/earth.md

@@ -5,7 +5,8 @@
 # Copy from earth
 scp -P 33224 earth@earth.cybernode.ai:/path/file /host/path/file
 # Copy to earth
-scp -P 33324 /host/path/file earth@earth.cybernode.ai:/path/file
+scp -P 33324 testnet/genesis.json earth@earth.cybernode.ai:/cyberdata/cyberd/config/genesis.json
+scp -P 33324 testnet/config.toml earth@earth.cybernode.ai:/cyberdata/cyberd/config/config.toml
 ```
 
 ## Reset cyberd