You can hack around the OnError function receiver but honestly it's very gross because you're limited in how much you can hook into the Colly logic (really you want to push onto the on http callback slice, but it's private)

I strongly suggest doing this outside of colly with a std http request + goquery instead of the below.

func (c *Client) GetPage(_ context.Context, id string) (*PageResult, error) {
	pageUrl := "http://google.com"
	col := colly.NewCollector()
	var pageModel *PageModel
	col.UserAgent = userAgent

	var err error

	col.OnError(func(res *colly.Response, collyErr error) {
		if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusGone {
			err = fmt.Errorf("invalid status code for page %s: %w", pageUrl, err)
			return
		}

		doc, err := goquery.NewDocumentFromReader(bytes.NewBuffer(res.Body))
		if err != nil {
			err = fmt.Errorf("could not parse response body: %w", err)
			return
		}

		doc.Find("script").Each(func(i int, s *goquery.Selection) {
		    if i == 0 {
                         pageModel = s.Text()
                     }
		})
	})

	_ = col.Visit(pageUrl)
	if err != nil {
		return nil, fmt.Errorf("could not visit %s: %w", pageUrl, err)
	}

	return &PageResult{
		Model: pageModel,
	}, nil
}

how to by pass c.OnError #799

Description

Metadata

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

Issue actions