runner

package
v0.0.0-...-a4de56b Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jan 15, 2026 License: MIT Imports: 24 Imported by: 0

Documentation

Index

Constants

View Source
const (
	RunModeFile = iota + 1
	RunModeDatabase
	RunModeDatabaseProduce
	RunModeInstallPlaywright
	RunModeAwsLambda
	RunModeAwsLambdaInvoker
	RunModeManager
	RunModeWorker
)

Variables

View Source
var (
	// Version is the current application version
	// It is injected at build time via -ldflags
	Version = "dev"

	// BuildDate is the timestamp of the build
	// It is injected at build time via -ldflags
	BuildDate = "unknown"

	// Commit is the git commit hash
	// It is injected at build time via -ldflags
	Commit = "none"
)
View Source
var (
	ErrInvalidRunMode = errors.New("invalid run mode")
)

Functions

func Banner()

func CreateSeedJobs

func CreateSeedJobs(
	fastmode bool,
	langCode string,
	r io.Reader,
	maxDepth int,
	email bool,
	geoCoordinates string,
	zoom int,
	radius float64,
	dedup deduper.Deduper,
	exitMonitor exiter.Exiter,
	extraReviews bool,
) (jobs []scrapemate.IJob, err error)

func CreateSeedJobsFromKeywords

func CreateSeedJobsFromKeywords(cfg SeedJobConfig) ([]scrapemate.IJob, error)

CreateSeedJobsFromKeywords creates seed jobs from a slice of keywords. This is a reusable wrapper for CreateSeedJobs that accepts []string instead of io.Reader. Used by both CLI and API (Dashboard).

func FormatGeoCoordinates

func FormatGeoCoordinates(lat, lon float64) string

FormatGeoCoordinates formats latitude and longitude into a string. Returns empty string if both are zero.

func LoadCustomWriter

func LoadCustomWriter(pluginDir, pluginName string) (scrapemate.ResultWriter, error)

func Telemetry

func Telemetry() tlmt.Telemetry

Types

type Config

type Config struct {
	Concurrency              int
	CacheDir                 string
	MaxDepth                 int
	InputFile                string
	ResultsFile              string
	JSON                     bool
	LangCode                 string
	Debug                    bool
	Dsn                      string
	ProduceOnly              bool
	ExitOnInactivityDuration time.Duration
	Email                    bool
	CustomWriter             string
	GeoCoordinates           string
	Zoom                     int
	RunMode                  int
	DisableTelemetry         bool
	AwsLamdbaRunner          bool
	DataFolder               string
	Proxies                  []string
	AwsAccessKey             string
	AwsSecretKey             string
	AwsRegion                string
	S3Uploader               S3Uploader
	S3Bucket                 string
	AwsLambdaInvoker         bool
	FunctionName             string
	AwsLambdaChunkSize       int
	FastMode                 bool
	Radius                   float64
	Addr                     string
	DisablePageReuse         bool
	ExtraReviews             bool
	LeadsDBAPIKey            string
	// Manager/Worker mode flags
	ManagerMode bool
	WorkerMode  bool
	ManagerURL  string
	WorkerID    string
	// StaticFolder is the path to static frontend files
	StaticFolder string

	// Redis configuration for job queue
	RedisURL  string
	RedisAddr string
	RedisPass string
	RedisDB   int

	// ProxyGate flags
	ProxyGateEnabled         bool
	ProxyGateAddr            string
	ProxyGateSources         []string
	ProxyGateRefreshInterval time.Duration

	// Email validation (Moribouncer)
	EmailValidatorURL string
	EmailValidatorKey string

	// Migration flags
	Migrate       bool // Run migration only, then exit
	MigrateStatus bool // Check migration status and exit
}

func ParseConfig

func ParseConfig() *Config

type Runner

type Runner interface {
	Run(context.Context) error
	Close(context.Context) error
}

type S3Uploader

type S3Uploader interface {
	Upload(ctx context.Context, bucketName, key string, body io.Reader) error
}

type SeedJobConfig

type SeedJobConfig struct {
	Keywords       []string
	FastMode       bool
	LangCode       string
	Depth          int
	Email          bool
	GeoCoordinates string // "lat,lon" or ""
	Zoom           int
	Radius         float64
	ExtraReviews   bool
	Dedup          deduper.Deduper
	ExitMonitor    exiter.Exiter
}

SeedJobConfig for creating seed jobs from API

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL