From 91bc5b48e569847c38365ea28f451617d1f77652 Mon Sep 17 00:00:00 2001 From: Tim Voronov Date: Mon, 24 Jan 2022 17:33:09 -0500 Subject: [PATCH 1/3] Fixed XPath eval in http driver for attrribute values --- pkg/drivers/http/element_test.go | 32 ++++++++++++++++++++++++++++++++ pkg/drivers/http/xpath.go | 15 +++++++++------ 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/pkg/drivers/http/element_test.go b/pkg/drivers/http/element_test.go index 4911cc3a..d004de57 100644 --- a/pkg/drivers/http/element_test.go +++ b/pkg/drivers/http/element_test.go @@ -6,6 +6,7 @@ import ( "github.com/MontFerret/ferret/pkg/drivers" "github.com/MontFerret/ferret/pkg/drivers/http" "github.com/MontFerret/ferret/pkg/runtime/values" + "github.com/MontFerret/ferret/pkg/runtime/values/types" "github.com/PuerkitoBio/goquery" . "github.com/smartystreets/goconvey/convey" "testing" @@ -447,5 +448,36 @@ func TestElement(t *testing.T) { So(err, ShouldBeNil) So(nt.String(), ShouldEqual, "[\"Album example for Bootstrap\"]") }) + + Convey("Attributes", func() { + buff := bytes.NewBuffer([]byte(`
`)) + godoc, err := goquery.NewDocumentFromReader(buff) + So(err, ShouldBeNil) + + doc, err := http.NewRootHTMLDocument(godoc, "localhost:9090") + So(err, ShouldBeNil) + + nt, err := doc.XPath(context.Background(), values.NewString("//a/@title")) + + So(err, ShouldBeNil) + So(nt.Type().String(), ShouldEqual, types.Array.String()) + So(nt.(*values.Array).First().Type().String(), ShouldEqual, types.String.String()) + So(nt.(*values.Array).First().String(), ShouldEqual, "30") + }) + + Convey("Element node", func() { + buff := bytes.NewBuffer([]byte(`
`)) + godoc, err := goquery.NewDocumentFromReader(buff) + So(err, ShouldBeNil) + + doc, err := http.NewRootHTMLDocument(godoc, "localhost:9090") + So(err, ShouldBeNil) + + nt, err := doc.XPath(context.Background(), values.NewString("//div")) + + So(err, ShouldBeNil) + So(nt.Type().String(), ShouldEqual, types.Array.String()) + So(nt.(*values.Array).First().Type().String(), ShouldEqual, drivers.HTMLElementType.String()) + }) }) } diff --git a/pkg/drivers/http/xpath.go b/pkg/drivers/http/xpath.go index 73258b92..540affd0 100644 --- a/pkg/drivers/http/xpath.go +++ b/pkg/drivers/http/xpath.go @@ -84,12 +84,15 @@ func EvalXPathTo(selection *goquery.Selection, expression string) (core.Value, e for res.MoveNext() { var item core.Value - node := res.Current().(*htmlquery.NodeNavigator).Current() - - if node.Type == html.TextNode { - item = values.NewString(node.Data) - } else { - i, err := parseXPathNode(node) + node := res.Current() + + switch node.NodeType() { + case xpath.TextNode: + item = values.NewString(node.Value()) + case xpath.AttributeNode: + item = values.NewString(node.Value()) + default: + i, err := parseXPathNode(node.(*htmlquery.NodeNavigator).Current()) if err != nil { return nil, err From fa7ee5f3d45ac8e839d3ef7346fbd1deed6e2e6d Mon Sep 17 00:00:00 2001 From: Tim Voronov Date: Mon, 24 Jan 2022 18:49:18 -0500 Subject: [PATCH 2/3] Added unit test to cover http XPath func eval --- pkg/drivers/http/element_test.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pkg/drivers/http/element_test.go b/pkg/drivers/http/element_test.go index d004de57..dd0cdcf0 100644 --- a/pkg/drivers/http/element_test.go +++ b/pkg/drivers/http/element_test.go @@ -449,6 +449,25 @@ func TestElement(t *testing.T) { So(nt.String(), ShouldEqual, "[\"Album example for Bootstrap\"]") }) + Convey("Func", func() { + buff := bytes.NewBuffer([]byte(doc)) + + buff.Write([]byte(doc)) + + doc, err := goquery.NewDocumentFromReader(buff) + + So(err, ShouldBeNil) + + el, err := http.NewHTMLElement(doc.Find("html")) + + So(err, ShouldBeNil) + + nt, err := el.XPath(context.Background(), values.NewString("count(//div)")) + + So(err, ShouldBeNil) + So(nt.Type().String(), ShouldEqual, types.Float.String()) + }) + Convey("Attributes", func() { buff := bytes.NewBuffer([]byte(`
`)) godoc, err := goquery.NewDocumentFromReader(buff) From 027ff34b796fc2104aad8b7659dec9d68be4ef1e Mon Sep 17 00:00:00 2001 From: Tim Voronov Date: Fri, 28 Jan 2022 12:29:44 -0500 Subject: [PATCH 3/3] Added integration test for covering XPath by CDP driver --- e2e/tests/dynamic/element/xpath/attrs.fql | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 e2e/tests/dynamic/element/xpath/attrs.fql diff --git a/e2e/tests/dynamic/element/xpath/attrs.fql b/e2e/tests/dynamic/element/xpath/attrs.fql new file mode 100644 index 00000000..fd7e8069 --- /dev/null +++ b/e2e/tests/dynamic/element/xpath/attrs.fql @@ -0,0 +1,8 @@ +LET url = @lab.cdn.dynamic +LET page = DOCUMENT(url, true) + +LET actual = XPATH(page, "//body/@class") + +T::NOT::EMPTY(actual) + +RETURN T::EQ(actual[0], "text-center") \ No newline at end of file