spider

type BaseSpiderConf struct {
	EnableCookie   bool
	EnableProxy    bool
	ProxyUrl       string
	DownloaderType string

	Cookie *cookiejar.Jar
}

type Context struct {
	Ctx      context.Context
	Cancel   context.CancelFunc
	Response *http.Response

	Temp map[string]interface{}
	Info map[string]string
	// contains filtered or unexported fields
}

type MiddlewareHandler func(ctx *Context)

type MiddlewareHandlerErr func(ctx *Context) error

type Option func(b *BaseSpiderConf)

type Resource struct {
	SpiderUniqueKey string

	*downloader.Request
	// contains filtered or unexported fields
}

type Schedule struct {
	ResourcePoolList chan Resource
	ConcurrentNum    int // 并发数量
	// contains filtered or unexported fields
}

type Spider struct {
	UniqueKey string // 唯一标识符
	STATUS    uint   // 状态

	Downloader    downloader.Downloader          // 下载器
	RuleHandlers  map[string][]MiddlewareHandler // 规则中间件
	CloseCallback func(s *Spider)                // 回调关闭
	// contains filtered or unexported fields
}