Documentation
¶
Index ¶
- Constants
- Variables
- func Banner()
- func CreateSeedJobs(fastmode bool, langCode string, r io.Reader, maxDepth int, email bool, ...) (jobs []scrapemate.IJob, err error)
- func CreateSeedJobsFromKeywords(cfg SeedJobConfig) ([]scrapemate.IJob, error)
- func FormatGeoCoordinates(lat, lon float64) string
- func LoadCustomWriter(pluginDir, pluginName string) (scrapemate.ResultWriter, error)
- func Telemetry() tlmt.Telemetry
- type Config
- type Runner
- type S3Uploader
- type SeedJobConfig
Constants ¶
View Source
const ( RunModeFile = iota + 1 RunModeDatabase RunModeDatabaseProduce RunModeInstallPlaywright RunModeAwsLambda RunModeAwsLambdaInvoker RunModeManager RunModeWorker )
Variables ¶
View Source
var ( // Version is the current application version // It is injected at build time via -ldflags Version = "dev" // BuildDate is the timestamp of the build // It is injected at build time via -ldflags BuildDate = "unknown" // Commit is the git commit hash // It is injected at build time via -ldflags Commit = "none" )
View Source
var (
ErrInvalidRunMode = errors.New("invalid run mode")
)
Functions ¶
func CreateSeedJobs ¶
func CreateSeedJobsFromKeywords ¶
func CreateSeedJobsFromKeywords(cfg SeedJobConfig) ([]scrapemate.IJob, error)
CreateSeedJobsFromKeywords creates seed jobs from a slice of keywords. This is a reusable wrapper for CreateSeedJobs that accepts []string instead of io.Reader. Used by both CLI and API (Dashboard).
func FormatGeoCoordinates ¶
FormatGeoCoordinates formats latitude and longitude into a string. Returns empty string if both are zero.
func LoadCustomWriter ¶
func LoadCustomWriter(pluginDir, pluginName string) (scrapemate.ResultWriter, error)
Types ¶
type Config ¶
type Config struct {
Concurrency int
CacheDir string
MaxDepth int
InputFile string
ResultsFile string
JSON bool
LangCode string
Debug bool
Dsn string
ProduceOnly bool
ExitOnInactivityDuration time.Duration
Email bool
CustomWriter string
GeoCoordinates string
Zoom int
RunMode int
DisableTelemetry bool
AwsLamdbaRunner bool
DataFolder string
Proxies []string
AwsAccessKey string
AwsSecretKey string
AwsRegion string
S3Uploader S3Uploader
S3Bucket string
AwsLambdaInvoker bool
FunctionName string
AwsLambdaChunkSize int
FastMode bool
Radius float64
Addr string
DisablePageReuse bool
ExtraReviews bool
LeadsDBAPIKey string
// Manager/Worker mode flags
ManagerMode bool
WorkerMode bool
ManagerURL string
WorkerID string
// StaticFolder is the path to static frontend files
StaticFolder string
// Redis configuration for job queue
RedisURL string
RedisAddr string
RedisPass string
RedisDB int
// ProxyGate flags
ProxyGateEnabled bool
ProxyGateAddr string
ProxyGateSources []string
ProxyGateRefreshInterval time.Duration
// Email validation (Moribouncer)
EmailValidatorURL string
EmailValidatorKey string
// Migration flags
Migrate bool // Run migration only, then exit
MigrateStatus bool // Check migration status and exit
}
func ParseConfig ¶
func ParseConfig() *Config
type S3Uploader ¶
Click to show internal directories.
Click to hide internal directories.