feat: add scanmalware search agent

README.md

@@ -52,6 +52,7 @@
   - **[Driftnet](https://driftnet.io)**
   - **[DayDayMap](https://www.daydaymap.com)**
   - **[NerdyData](https://www.nerdydata.com/?utm_source=projectdiscovery/uncover)**
+  - **[ScanMalware](https://scanmalware.com)**
 - Multiple API key input support
 - Automatic API key randomization
 - **stdin** / **stdout** support for input
@@ -79,12 +80,13 @@ Usage:
 Flags:
 INPUT:
    -q, -query string[]   search query, supports: stdin,file,config input (example: -q 'example query', -q 'query.txt')
-   -e, -engine string[]  search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,criminalip,publicwww,hunterhow,google,driftnet,daydaymap) (default shodan)
+   -e, -engine string[]  search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,criminalip,publicwww,hunterhow,google,driftnet,daydaymap,scanmalware) (default shodan)
    -asq, -awesome-search-queries string[]  use awesome search queries to discover exposed assets on the internet (example: -asq 'jira')
 
 SEARCH-ENGINE:
    -s, -shodan string[]       search query for shodan (example: -shodan 'query.txt')
    -sd, -shodan-idb string[]  search query for shodan-idb (example: -shodan-idb 'query.txt')
+   -sm, -scanmalware string[] search query for scanmalware (example: -scanmalware 'domain:*.example.com')
    -ff, -fofa string[]        search query for fofa (example: -fofa 'query.txt')
    -cs, -censys string[]      search query for censys (example: -censys 'query.txt')
    -qk, -quake string[]       search query for quake (example: -quake 'query.txt')
@@ -485,6 +487,7 @@ https://129.206.117.248
 - `query` flag supports **all and only filters supported by search engine.**
 - results are limited to `100` as default and can be increased with `limit` flag.
 - `shodan-idb` API doesn't requires an API key and works out of the box.
+- `scanmalware` API doesn't require an API key either. It searches an archive of sandboxed URL scans, so it returns URLs and hostnames rather than `ip:port`; `-f host` or `-f url` is the useful output here.
 - `shodan-idb` API is used as **default** engine when **IP/CIDR** is provided as input.
 
 -----

runner/options.go

@@ -66,6 +66,7 @@ type Options struct {
 	GreyNoise            goflags.StringSlice
 	Daydaymap            goflags.StringSlice
 	NerdyData            goflags.StringSlice
+	ScanMalware          goflags.StringSlice
 	DisableUpdateCheck   bool
 	CheckCredits         bool
 }
@@ -78,7 +79,7 @@ func ParseOptions() *Options {
 
 	flagSet.CreateGroup("input", "Input",
 		flagSet.StringSliceVarP(&options.Query, "query", "q", nil, "search query, supports: stdin,file,config input (example: -q 'example query', -q 'query.txt')", goflags.FileStringSliceOptions),
-		flagSet.StringSliceVarP(&options.Engine, "engine", "e", nil, "search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,publicwww,criminalip,hunterhow,google,odin,binaryedge,onyphe,driftnet,greynoise,daydaymap,nerdydata) (default shodan)", goflags.FileNormalizedStringSliceOptions),
+		flagSet.StringSliceVarP(&options.Engine, "engine", "e", nil, "search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,publicwww,criminalip,hunterhow,google,odin,binaryedge,onyphe,driftnet,greynoise,daydaymap,nerdydata,scanmalware) (default shodan)", goflags.FileNormalizedStringSliceOptions),
 		flagSet.StringSliceVarP(&options.AwesomeSearchQueries, "awesome-search-queries", "asq", nil, "use awesome search queries to discover exposed assets on the internet (example: -asq 'jira')", goflags.FileStringSliceOptions),
 	)
 
@@ -102,6 +103,7 @@ func ParseOptions() *Options {
 		flagSet.StringSliceVarP(&options.GreyNoise, "greynoise", "gn", nil, "search query for greynoise (example: -greynoise 'query.txt')", goflags.FileStringSliceOptions),
 		flagSet.StringSliceVarP(&options.Daydaymap, "daydaymap", "ddm", nil, "search query for daydaymap (example: -daydaymap 'query.txt')", goflags.FileStringSliceOptions),
 		flagSet.StringSliceVarP(&options.NerdyData, "nerdydata", "nd", nil, "search query for NerdyData (example: -nerdydata 'query.txt')", goflags.FileStringSliceOptions),
+		flagSet.StringSliceVarP(&options.ScanMalware, "scanmalware", "sm", nil, "search query for ScanMalware (example: -scanmalware 'domain:*.example.com')", goflags.FileStringSliceOptions),
 		flagSet.BoolVarP(&options.CheckCredits, "check-credits", "cc", false, "check remaining credits for configured providers"),
 		flagSet.BoolVar(&options.CheckCredits, "credits", false, "check remaining credits for configured providers"),
 	)
@@ -183,7 +185,8 @@ func ParseOptions() *Options {
 		len(options.Driftnet),
 		len(options.GreyNoise),
 		len(options.Daydaymap),
-		len(options.NerdyData)) {
+		len(options.NerdyData),
+		len(options.ScanMalware)) {
 		if !options.CheckCredits {
 			options.Engine = append(options.Engine, "shodan")
 		}
@@ -283,7 +286,8 @@ func (options *Options) validateOptions() error {
 		len(options.Driftnet),
 		len(options.GreyNoise),
 		len(options.Daydaymap),
-		len(options.NerdyData)) {
+		len(options.NerdyData),
+		len(options.ScanMalware)) {
 		return errors.New("no query provided")
 	}
 
@@ -309,7 +313,8 @@ func (options *Options) validateOptions() error {
 		len(options.Driftnet),
 		len(options.GreyNoise),
 		len(options.Daydaymap),
-		len(options.NerdyData)) {
+		len(options.NerdyData),
+		len(options.ScanMalware)) {
 		return errors.New("no engine specified")
 	}
 
@@ -356,6 +361,7 @@ func appendAllQueries(options *Options) {
 	appendQuery(options, "greynoise", options.GreyNoise...)
 	appendQuery(options, "daydaymap", options.Daydaymap...)
 	appendQuery(options, "nerdydata", options.NerdyData...)
+	appendQuery(options, "scanmalware", options.ScanMalware...)
 }
 
 func (options *Options) useAwesomeSearchQueries(awesomeSearchQueries []string) error {

sources/agent/scanmalware/response.go

@@ -0,0 +1,22 @@
+package scanmalware
+
+// smqlResponse is one page of the ScanMalware archive search.
+type smqlResponse struct {
+	Query   string       `json:"query"`
+	Results []smqlResult `json:"results"`
+}
+
+// smqlResult is one scanned page. The archive records what a browser saw, so
+// there is no IP or port here: `ip_count` and the ASN fields describe the
+// addresses the scan resolved, not a single listening service.
+type smqlResult struct {
+	ScanID     string   `json:"scan_id"`
+	URL        string   `json:"url"`
+	FinalURL   string   `json:"final_url"`
+	Title      string   `json:"title"`
+	Status     string   `json:"status"`
+	PrimaryASN string   `json:"primary_asn"`
+	ASNNames   []string `json:"asn_names"`
+	Countries  []string `json:"countries"`
+	IPCount    int      `json:"ip_count"`
+}

sources/agent/scanmalware/scanmalware.go

@@ -0,0 +1,126 @@
+// Package scanmalware queries the ScanMalware archive of sandboxed URL scans.
+//
+// The query is SMQL, ScanMalware's own search syntax, passed through unchanged,
+// the same way the fofa and shodan agents pass theirs. Examples:
+//
+//	uncover -scanmalware 'domain:*.example.com'
+//	uncover -scanmalware 'asn:15169'
+//
+// The API is anonymous, so this agent runs on a fresh install with no
+// provider-config.yaml, like shodan-idb.
+package scanmalware
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+
+	"github.com/projectdiscovery/uncover/sources"
+)
+
+var (
+	// URL is a var rather than a const so tests can point it at httptest.
+	URL = "https://scanmalware.com/api/v1/search/smql?q=%s&limit=%d&page=%d"
+)
+
+const (
+	// pageSize is the maximum the endpoint accepts. It rejects larger values
+	// rather than clamping them.
+	pageSize = 100
+	// maxPages bounds a broad query so one agent cannot stall a whole run.
+	maxPages = 100
+	// userAgent identifies uncover to ScanMalware, so this traffic is
+	// attributable rather than looking like scraping.
+	userAgent = "scanmalware-uncover/1.0 (+https://github.com/projectdiscovery/uncover)"
+)
+
+type Agent struct{}
+
+func (agent *Agent) Name() string {
+	return "scanmalware"
+}
+
+func (agent *Agent) Query(ctx context.Context, session *sources.Session, query *sources.Query) (chan sources.Result, error) {
+	results := make(chan sources.Result)
+
+	go func() {
+		defer close(results)
+		agent.query(ctx, session, query, results)
+	}()
+
+	return results, nil
+}
+
+func (agent *Agent) query(ctx context.Context, session *sources.Session, query *sources.Query, results chan sources.Result) {
+	sent := 0
+
+	for page := 1; page <= maxPages; page++ {
+		if ctx.Err() != nil {
+			return
+		}
+
+		resp, err := agent.queryURL(ctx, session, query.Query, page)
+		if err != nil {
+			sources.SendResult(ctx, results, sources.Result{Source: agent.Name(), Error: err})
+			return
+		}
+
+		var body smqlResponse
+		err = json.NewDecoder(resp.Body).Decode(&body)
+		if closeErr := resp.Body.Close(); closeErr != nil && err == nil {
+			err = closeErr
+		}
+		if err != nil {
+			sources.SendResult(ctx, results, sources.Result{Source: agent.Name(), Error: err})
+			return
+		}
+
+		for _, row := range body.Results {
+			if sent >= query.Limit {
+				return
+			}
+
+			result := sources.Result{Source: agent.Name(), Url: row.URL}
+			result.Host = hostOf(row.URL)
+			if result.Host == "" {
+				result.Host = row.URL
+			}
+			result.Raw, _ = json.Marshal(row)
+
+			if !sources.SendResult(ctx, results, result) {
+				return
+			}
+			sent++
+		}
+
+		// A short page is the last page.
+		if len(body.Results) < pageSize {
+			return
+		}
+	}
+}
+
+func (agent *Agent) queryURL(ctx context.Context, session *sources.Session, smql string, page int) (*http.Response, error) {
+	requestURL := fmt.Sprintf(URL, url.QueryEscape(smql), pageSize, page)
+
+	request, err := sources.NewHTTPRequest(ctx, http.MethodGet, requestURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	request.Header.Set("User-Agent", userAgent)
+	request.Header.Set("x-uncover", "1")
+
+	return session.Do(request, agent.Name())
+}
+
+// hostOf returns the hostname of a URL, or an empty string when it cannot be
+// parsed. The archive stores what was submitted, which is not always well formed.
+func hostOf(rawURL string) string {
+	parsed, err := url.Parse(rawURL)
+	if err != nil {
+		return ""
+	}
+	return parsed.Hostname()
+}